diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 9366cc0..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,48 +0,0 @@ -cmake_minimum_required(VERSION 3.14) -project(ml VERSION 0.1.0 LANGUAGES CXX) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") -set(CMAKE_CXX_FLAGS_RELEASE "-O2") - -add_library(tensor_engine STATIC - src/tensor/cpu_engine.cpp -) -target_include_directories(tensor_engine PUBLIC src) - -add_library(frame STATIC - src/frame/frame.h -) -target_include_directories(frame PUBLIC src) -target_link_libraries(frame PUBLIC tensor_engine) - -add_executable(tensor_benchmark - src/tensor/tensor_benchmark.cpp -) -target_link_libraries(tensor_benchmark tensor_engine) - -add_executable(frame_example - src/frame/frame_example.cpp -) -target_link_libraries(frame_example frame) - -enable_testing() - -include(FetchContent) -FetchContent_Declare( - googletest - URL https://github.com/google/googletest/archive/refs/tags/v1.14.0.zip -) -set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) -FetchContent_MakeAvailable(googletest) - -add_executable(tensor_test src/tensor/tensor_test.cpp) -target_link_libraries(tensor_test tensor_engine GTest::gtest_main) -include(GoogleTest) -gtest_discover_tests(tensor_test) - -add_executable(frame_test src/frame/frame_test.cpp) -target_link_libraries(frame_test frame GTest::gtest_main) -include(GoogleTest) -gtest_discover_tests(frame_test) diff --git a/Makefile b/Makefile index 075c55c..f6cc0ef 100644 --- a/Makefile +++ b/Makefile @@ -8,39 +8,32 @@ INCLUDES = -I$(SRCDIR) TENSOR_ENGINE_SRC = $(SRCDIR)/tensor/cpu_engine.cpp TENSOR_BENCHMARK = $(BUILDDIR)/tensor_benchmark -EXAMPLE_FILES := $(shell find $(SRCDIR) -name "*_example.cpp" 2>/dev/null | grep -v tensor) -BENCHMARK_FILES := $(shell find $(SRCDIR) -name "*_benchmark.cpp" 2>/dev/null | grep -v tensor) -ALGO_TEST_FILES := $(shell find $(SRCDIR) -name "*_test.cpp" 2>/dev/null | grep -v tensor) - -ALGORITHMS := $(sort $(patsubst %_example,%,$(patsubst %_benchmark,%,$(basename $(notdir $(EXAMPLE_FILES) $(BENCHMARK_FILES) $(ALGO_TEST_FILES)))))) +ALGO_SRCS := $(shell find $(SRCDIR)/supervised -path "*/linear_square_error/*.cpp" ! -name "*_test.cpp" ! -name "*_example.cpp" ! -name "*_benchmark.cpp" 2>/dev/null) +ALGO_NAMES := $(sort $(basename $(notdir $(ALGO_SRCS)))) ALGO_SET := $(filter command line environment,$(origin ALGO)) ifeq ($(ALGO),) - ALGO := $(ALGORITHMS) + ALGO := $(ALGO_NAMES) endif -VALID_ALGO := $(filter $(ALGO), $(ALGORITHMS)) -INVALID_ALGO := $(filter-out $(ALGORITHMS), $(ALGO)) +VALID_ALGO := $(filter $(ALGO), $(ALGO_NAMES)) +INVALID_ALGO := $(filter-out $(ALGO), $(ALGO_NAMES)) $(if $(INVALID_ALGO), $(warning Unknown algorithms: $(INVALID_ALGO))) ALGO_EXAMPLES := $(addprefix $(BUILDDIR)/,$(addsuffix _example,$(VALID_ALGO))) ALGO_BENCHMARKS := $(addprefix $(BUILDDIR)/,$(addsuffix _benchmark,$(VALID_ALGO))) ALGO_TESTS := $(addprefix $(BUILDDIR)/,$(addsuffix _test,$(VALID_ALGO))) -find_algo_src = $(firstword $(shell find $(SRCDIR) -name "$(1).cpp" 2>/dev/null)) - -# ---- Test infrastructure ---- +find_algo_src = $(shell find $(SRCDIR)/supervised -name "$(1).cpp" 2>/dev/null | head -1) -TEST_FILES := $(shell find $(SRCDIR) -name "*_test.cpp" 2>/dev/null) -TEST_BINS := $(patsubst $(SRCDIR)/%.cpp,$(BUILDDIR)/%,$(TEST_FILES)) +TEST_FILES := $(shell find $(SRCDIR)/supervised -name "*_test.cpp" 2>/dev/null) +TEST_BINS := $(ALGO_TESTS) GTEST_CXXFLAGS = -std=c++17 -O2 -Wall -Wextra -I$(SRCDIR) -I/usr/local/include GTEST_LDFLAGS = -L/usr/local/lib -lgtest -lgtest_main -lpthread -# ---- Build rules (real targets under build/) ---- - -.PHONY: all clean list help test $(ALGORITHMS) +.PHONY: all clean list help test $(ALGO_NAMES) ifeq ($(ALGO_SET),) all: $(TENSOR_BENCHMARK) $(ALGO_EXAMPLES) $(ALGO_BENCHMARKS) @@ -59,43 +52,52 @@ $(TENSOR_BENCHMARK): $(SRCDIR)/tensor/tensor_benchmark.cpp $(TENSOR_ENGINE_SRC) $(CXX) $(CXXFLAGS) $(INCLUDES) $^ -o $@ $(BUILDDIR)/%_example: $(TENSOR_ENGINE_SRC) - $(eval EXAMPLE_SRC := $(call find_algo_src,$(*F)_example)) - $(eval ALGO_SRC := $(call find_algo_src,$(*F))) + $(eval algo := $(*F)) + $(eval EXAMPLE_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo)_example.cpp" 2>/dev/null | head -1)) + $(eval ALGO_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo).cpp" 2>/dev/null | grep -v example | grep -v benchmark | grep -v test | head -1)) @if [ -z "$(EXAMPLE_SRC)" ]; then \ - echo "Error: Source file for $(*F)_example not found"; exit 1; fi + echo "Error: Example source for $(algo) not found"; exit 1; fi @if [ -z "$(ALGO_SRC)" ]; then \ - echo "Error: Source file for $(*F) not found"; exit 1; fi + echo "Error: Algorithm source for $(algo) not found"; exit 1; fi @mkdir -p $(BUILDDIR) - $(CXX) $(CXXFLAGS) $(INCLUDES) $(EXAMPLE_SRC) $(ALGO_SRC) $< -o $@ + $(CXX) $(CXXFLAGS) $(INCLUDES) $(EXAMPLE_SRC) $(ALGO_SRC) $(TENSOR_ENGINE_SRC) -o $@ $(BUILDDIR)/%_benchmark: $(TENSOR_ENGINE_SRC) - $(eval BENCHMARK_SRC := $(call find_algo_src,$(*F)_benchmark)) - $(eval ALGO_SRC := $(call find_algo_src,$(*F))) + $(eval algo := $(*F)) + $(eval BENCHMARK_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo)_benchmark.cpp" 2>/dev/null | head -1)) + $(eval ALGO_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo).cpp" 2>/dev/null | grep -v example | grep -v benchmark | grep -v test | head -1)) @if [ -z "$(BENCHMARK_SRC)" ]; then \ - echo "Error: Source file for $(*F)_benchmark not found"; exit 1; fi + echo "Error: Benchmark source for $(algo) not found"; exit 1; fi @if [ -z "$(ALGO_SRC)" ]; then \ - echo "Error: Source file for $(*F) not found"; exit 1; fi + echo "Error: Algorithm source for $(algo) not found"; exit 1; fi @mkdir -p $(BUILDDIR) - $(CXX) $(CXXFLAGS) $(INCLUDES) $(BENCHMARK_SRC) $(ALGO_SRC) $< -o $@ - -$(BUILDDIR)/%_test: $(SRCDIR)/%_test.cpp $(TENSOR_ENGINE_SRC) - @mkdir -p $(dir $@) - $(CXX) $(GTEST_CXXFLAGS) $< $(TENSOR_ENGINE_SRC) $(GTEST_LDFLAGS) -o $@ + $(CXX) $(CXXFLAGS) $(INCLUDES) $(BENCHMARK_SRC) $(ALGO_SRC) $(TENSOR_ENGINE_SRC) -o $@ + +$(BUILDDIR)/%_test: $(TENSOR_ENGINE_SRC) + $(eval algo := $(*F)) + $(eval TEST_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo)_test.cpp" 2>/dev/null | head -1)) + $(eval ALGO_SRC := $(shell find $(SRCDIR)/supervised -name "$(algo).cpp" 2>/dev/null | grep -v example | grep -v benchmark | grep -v test | head -1)) + @if [ -z "$(TEST_SRC)" ]; then \ + echo "Error: Test source for $(algo) not found"; exit 1; fi + @if [ -z "$(ALGO_SRC)" ]; then \ + echo "Error: Algorithm source for $(algo) not found"; exit 1; fi + @mkdir -p $(BUILDDIR) + $(CXX) $(GTEST_CXXFLAGS) $(TEST_SRC) $(ALGO_SRC) $(TENSOR_ENGINE_SRC) $(GTEST_LDFLAGS) -o $@ -$(ALGORITHMS): +$(ALGO_NAMES): @$(MAKE) --no-print-directory ALGO=$@ -$(foreach algo,$(ALGORITHMS),\ +$(foreach algo,$(ALGO_NAMES),\ $(eval .PHONY: $(algo)_test)\ $(eval $(algo)_test: ; @$$(MAKE) --no-print-directory $$(BUILDDIR)/$(algo)_test)\ ) -$(foreach algo,$(ALGORITHMS),\ +$(foreach algo,$(ALGO_NAMES),\ $(eval .PHONY: $(algo)_example)\ $(eval $(algo)_example: ; @$$(MAKE) --no-print-directory $$(BUILDDIR)/$(algo)_example)\ ) -$(foreach algo,$(ALGORITHMS),\ +$(foreach algo,$(ALGO_NAMES),\ $(eval .PHONY: $(algo)_benchmark)\ $(eval $(algo)_benchmark: ; @$$(MAKE) --no-print-directory $$(BUILDDIR)/$(algo)_benchmark)\ ) @@ -131,11 +133,11 @@ endif list: @echo "Available algorithms:" - @for algo in $(ALGORITHMS); do echo " - $$algo"; done - @if [ -z "$(ALGORITHMS)" ]; then echo " (none found)"; fi + @for algo in $(ALGO_NAMES); do echo " - $$algo"; done + @if [ -z "$(ALGO_NAMES)" ]; then echo " (none found)"; fi @echo "" @echo "Available tests:" - @for f in $(notdir $(TEST_FILES)); do echo " - $$f"; done + @for f in $(notdir $(TEST_FILES)); do echo " - $$(basename $$f .cpp)"; done @if [ -z "$(TEST_FILES)" ]; then echo " (none found)"; fi @echo "" @echo "Usage:" @@ -165,14 +167,14 @@ help: @echo "Examples:" @echo " make # Build everything" @echo " make test # Build and run all tests" - @echo " make test ALGO=linear_regression # Run linear_regression tests only" - @echo " make linear_regression_test # Build and run linear_regression test" - @echo " make linear_regression # Build linear regression only" - @echo " make ALGO=linear_regression # Same as above" - @echo " make linear_regression_example # Build only example" + @echo " make test ALGO=linear_square_error # Run linear_square_error tests only" + @echo " make linear_square_error_test # Build and run linear_square_error test" + @echo " make linear_square_error # Build linear square error only" + @echo " make ALGO=linear_square_error # Same as above" + @echo " make linear_square_error_example # Build only example" @echo "" @echo "Currently available algorithms:" - @for algo in $(ALGORITHMS); do echo " - $$algo"; done + @for algo in $(ALGO_NAMES); do echo " - $$algo"; done clean: rm -rf $(BUILDDIR) diff --git a/src/supervised/regression/linear_square_error/linear_square_error.cpp b/src/supervised/regression/linear_square_error/linear_square_error.cpp new file mode 100644 index 0000000..578718e --- /dev/null +++ b/src/supervised/regression/linear_square_error/linear_square_error.cpp @@ -0,0 +1,41 @@ +#include "linear_square_error.h" + +namespace ml { + +template +void LinearSquareError::fit(Tensor& X, Tensor& y, size_t n_samples, size_t n_features) { + Tensor X_aug(engine_, n_samples, n_features + 1); + for (size_t i = 0; i < n_samples; ++i) { + X_aug(i, 0) = T(1); + for (size_t j = 0; j < n_features; ++j) { + X_aug(i, j + 1) = X[i * n_features + j]; + } + } + + Tensor y_col = y.reshape(n_samples, 1); + + Tensor Xt = X_aug.transpose(); + Tensor XtX = Xt.matmul(X_aug); + Tensor XtX_inv = XtX.inv(); + Tensor Xty = Xt.matmul(y_col); + Tensor result = XtX_inv.matmul(Xty); + + bias_ = result(0, 0); + weights_ = Tensor(engine_, n_features); + for (size_t j = 0; j < n_features; ++j) { + weights_[j] = result(j + 1, 0); + } +} + +template +Tensor LinearSquareError::predict(Tensor& X, size_t n_samples, size_t n_features) { + Tensor y_pred(engine_, n_samples); + engine_.gemv(Trans::No, n_samples, n_features, T(1), X.data(), weights_.data(), T(0), y_pred.data()); + engine_.add_scalar(y_pred.data(), bias_, y_pred.data(), n_samples); + return y_pred; +} + +template class LinearSquareError; +template class LinearSquareError; + +} // namespace ml diff --git a/src/supervised/regression/linear_square_error/linear_square_error.h b/src/supervised/regression/linear_square_error/linear_square_error.h new file mode 100644 index 0000000..d2f03b3 --- /dev/null +++ b/src/supervised/regression/linear_square_error/linear_square_error.h @@ -0,0 +1,24 @@ +#pragma once + +#include "../../../tensor/tensor.h" + +namespace ml { + +template +class LinearSquareError { +public: + LinearSquareError(TensorEngine& engine) : engine_(engine), bias_(T(0)) {} + + void fit(Tensor& X, Tensor& y, size_t n_samples, size_t n_features); + Tensor predict(Tensor& X, size_t n_samples, size_t n_features); + + const Tensor& weights() const { return weights_; } + T bias() const { return bias_; } + +private: + TensorEngine& engine_; + Tensor weights_{engine_, 0}; + T bias_; +}; + +} // namespace ml diff --git a/src/supervised/regression/linear_square_error/linear_square_error_benchmark.cpp b/src/supervised/regression/linear_square_error/linear_square_error_benchmark.cpp new file mode 100644 index 0000000..82ed1a5 --- /dev/null +++ b/src/supervised/regression/linear_square_error/linear_square_error_benchmark.cpp @@ -0,0 +1,47 @@ +#include "linear_square_error.h" +#include "../../../tensor/cpu_engine.h" +#include +#include +#include + +using namespace ml; +using Clock = std::chrono::high_resolution_clock; + +void benchmark_lr(size_t n_samples, size_t n_features) { + CpuTensorEngine engine; + + std::mt19937 rng(42); + std::normal_distribution dist(0.0f, 1.0f); + + Tensor X(engine, n_samples * n_features); + Tensor y(engine, n_samples); + + for (size_t i = 0; i < n_samples * n_features; ++i) + X[i] = dist(rng); + for (size_t i = 0; i < n_samples; ++i) + y[i] = dist(rng); + + LinearSquareError model(engine); + + auto start = Clock::now(); + model.fit(X, y, n_samples, n_features); + auto end = Clock::now(); + double ms = std::chrono::duration(end - start).count(); + + printf(" samples=%8zu features=%4zu time=%8.2f ms\n", + n_samples, n_features, ms); +} + +int main() { + printf("=== Linear Square Error Benchmark ===\n\n"); + + printf("Scaling samples (1 feature):\n"); + for (size_t n : {100, 500, 1000, 5000, 10000, 50000}) + benchmark_lr(n, 1); + + printf("\nScaling features (1000 samples):\n"); + for (size_t f : {1, 5, 10, 50, 100, 500}) + benchmark_lr(1000, f); + + return 0; +} diff --git a/src/supervised/regression/linear_square_error/linear_square_error_example.cpp b/src/supervised/regression/linear_square_error/linear_square_error_example.cpp new file mode 100644 index 0000000..4d015ab --- /dev/null +++ b/src/supervised/regression/linear_square_error/linear_square_error_example.cpp @@ -0,0 +1,56 @@ +#include "linear_square_error.h" +#include "../../../tensor/cpu_engine.h" +#include +#include +#include + +using namespace ml; +using Clock = std::chrono::high_resolution_clock; + +int main() { + CpuTensorEngine engine; + + size_t n_samples = 100; + size_t n_features = 1; + float true_w = 3.0f; + float true_b = 2.0f; + + std::mt19937 rng(42); + std::normal_distribution noise(0.0f, 0.1f); + + Tensor X(engine, n_samples * n_features); + Tensor y(engine, n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + float x_val = static_cast(i) / n_samples; + X[i] = x_val; + y[i] = true_w * x_val + true_b + noise(rng); + } + + LinearSquareError model(engine); + + auto start = Clock::now(); + model.fit(X, y, n_samples, n_features); + auto end = Clock::now(); + double ms = std::chrono::duration(end - start).count(); + + printf("=== Linear Square Error (Least Squares) ===\n\n"); + printf("True: y = %.1f * x + %.1f\n", true_w, true_b); + printf("Learned: y = %.4f * x + %.4f\n", model.weights()[0], model.bias()); + printf("Training time: %.2f ms\n\n", ms); + + Tensor preds = model.predict(X, n_samples, n_features); + + float mse = 0.0f; + float rss = 0.0f; + for (size_t i = 0; i < n_samples; ++i) { + float diff = preds[i] - y[i]; + mse += diff * diff; + rss += diff * diff; + } + mse /= n_samples; + printf("MSE on training data: %.6f\n", mse); + printf("RSS (Residual Sum of Squares): %.6f\n", rss); + + return 0; +} diff --git a/src/supervised/regression/linear_square_error/linear_square_error_test.cpp b/src/supervised/regression/linear_square_error/linear_square_error_test.cpp new file mode 100644 index 0000000..ae1f234 --- /dev/null +++ b/src/supervised/regression/linear_square_error/linear_square_error_test.cpp @@ -0,0 +1,125 @@ +#include +#include +#include + +#include "tensor/tensor.h" +#include "tensor/cpu_engine.h" +#include "linear_square_error.h" + +using namespace ml; + +static constexpr float kTol = 1e-4f; + +class LinearSquareErrorTest : public ::testing::Test { +protected: + CpuTensorEngine eng; +}; + +TEST_F(LinearSquareErrorTest, SimpleLinearFit) { + size_t n_samples = 10; + size_t n_features = 1; + float true_w = 3.0f; + float true_b = 2.0f; + + Tensor X(eng, n_samples * n_features); + Tensor y(eng, n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + float x_val = static_cast(i) / n_samples; + X[i] = x_val; + y[i] = true_w * x_val + true_b; + } + + LinearSquareError model(eng); + model.fit(X, y, n_samples, n_features); + + EXPECT_NEAR(model.weights()[0], true_w, kTol); + EXPECT_NEAR(model.bias(), true_b, kTol); +} + +TEST_F(LinearSquareErrorTest, MultiFeature) { + size_t n_samples = 50; + size_t n_features = 2; + float true_w1 = 2.5f; + float true_w2 = -1.5f; + float true_b = 1.0f; + + std::mt19937 rng(42); + std::uniform_real_distribution dist(0.0f, 1.0f); + + Tensor X(eng, n_samples * n_features); + Tensor y(eng, n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + float x1 = dist(rng); + float x2 = dist(rng); + X[i * n_features] = x1; + X[i * n_features + 1] = x2; + y[i] = true_w1 * x1 + true_w2 * x2 + true_b; + } + + LinearSquareError model(eng); + model.fit(X, y, n_samples, n_features); + + EXPECT_NEAR(model.weights()[0], true_w1, kTol); + EXPECT_NEAR(model.weights()[1], true_w2, kTol); +} + +TEST_F(LinearSquareErrorTest, RSS_Calculation) { + size_t n_samples = 10; + size_t n_features = 1; + + Tensor X(eng, n_samples * n_features); + Tensor y(eng, n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + X[i] = static_cast(i); + y[i] = static_cast(i * 2 + 5); + } + + LinearSquareError model(eng); + model.fit(X, y, n_samples, n_features); + + Tensor preds = model.predict(X, n_samples, n_features); + + float rss = 0.0f; + for (size_t i = 0; i < n_samples; ++i) { + float residual = y[i] - preds[i]; + rss += residual * residual; + } + + EXPECT_LT(rss, kTol); +} + +TEST_F(LinearSquareErrorTest, MSE_Training) { + size_t n_samples = 100; + size_t n_features = 1; + float true_w = 2.0f; + float true_b = 1.0f; + + std::mt19937 rng(123); + std::normal_distribution noise(0.0f, 0.1f); + + Tensor X(eng, n_samples * n_features); + Tensor y(eng, n_samples); + + for (size_t i = 0; i < n_samples; ++i) { + float x_val = static_cast(i) / n_samples; + X[i] = x_val; + y[i] = true_w * x_val + true_b + noise(rng); + } + + LinearSquareError model(eng); + model.fit(X, y, n_samples, n_features); + + Tensor preds = model.predict(X, n_samples, n_features); + + float mse = 0.0f; + for (size_t i = 0; i < n_samples; ++i) { + float diff = preds[i] - y[i]; + mse += diff * diff; + } + mse /= n_samples; + + EXPECT_LT(mse, 1.0f); +}