From 20580372a32031ef503376c887add49cfd43b201 Mon Sep 17 00:00:00 2001 From: Shreyash Date: Sun, 31 Aug 2025 22:34:48 +0530 Subject: [PATCH 1/4] feat: initialise activation library functions --- nn/include/activation.h | 31 +++++ nn/src/activation/activation.c | 235 +++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 nn/include/activation.h create mode 100644 nn/src/activation/activation.c diff --git a/nn/include/activation.h b/nn/include/activation.h new file mode 100644 index 0000000..919ca2a --- /dev/null +++ b/nn/include/activation.h @@ -0,0 +1,31 @@ +#pragma once + +#include "linalg.h" + +//============================ +// Activation Functions +//============================ + +// We'll need the derivatives of these functions in the backprop algo +// So we'll define functions for derivatives here too. + +Matrix* sigmoid(Matrix* m); +Matrix* sigmoid_prime(Matrix* m); + +Matrix* relu(Matrix* m); +Matrix* relu_prime(Matrix* m); + +Matrix* tanh_activation(Matrix* m); +Matrix* tanh_prime(Matrix* m); + +Matrix* leaky_relu(Matrix* m); +Matrix* leaky_relu_prime(Matrix* m); + +Matrix* sign_activation(Matrix* m); +Matrix* sign_prime(Matrix* m); + +Matrix* identity_activation(Matrix* m); +Matrix* identity_prime(Matrix* m); + +Matrix* hard_tanh(Matrix* m); +Matrix* hard_tanh_prime(Matrix* m); diff --git a/nn/src/activation/activation.c b/nn/src/activation/activation.c new file mode 100644 index 0000000..3d2d34e --- /dev/null +++ b/nn/src/activation/activation.c @@ -0,0 +1,235 @@ +#include +#include +#include +#include + +#include "activation.h" +#include "linalg.h" +#include "utils.h" + +//============================ +// Sigmoid Activation +//============================ + +Matrix* sigmoid(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying sigmoid activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + result->matrix_data[i] = 1.0 / (1.0 + exp(-m->matrix_data[i])); + } + return result; +} + +Matrix* sigmoid_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying sigmoid_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + double sigmoid_val = 1.0 / (1.0 + exp(-m->matrix_data[i])); + result->matrix_data[i] = sigmoid_val * (1.0 - sigmoid_val); + } + return result; +} + +//============================ +// ReLU Activation +//============================ + +Matrix* relu(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying ReLU activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = m->matrix_data[i]; + } else { + result->matrix_data[i] = 0; + } + } + return result; +} + +Matrix* relu_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying ReLU_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = 1; + } else { + result->matrix_data[i] = 0; + } + } + return result; +} + +//============================ +// Tanh Activation +//============================ + +Matrix* tanh_activation(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Tanh activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + result->matrix_data[i] = tanh(m->matrix_data[i]); + } + return result; +} + +Matrix* tanh_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Tanh_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + double tanh_val = tanh(m->matrix_data[i]); + result->matrix_data[i] = 1.0 - pow(tanh_val, 2); + } + return result; +} + +//============================ +// Leaky ReLU Activation +//============================ + +Matrix* leaky_relu(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Leaky ReLU activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = m->matrix_data[i]; + } else { + // Common practice to use a small alpha, like 0.01 + result->matrix_data[i] = 0.01 * m->matrix_data[i]; + } + } + return result; +} + +Matrix* leaky_relu_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Leaky ReLU_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = 1; + } else { + result->matrix_data[i] = 0.01; + } + } + return result; +} + +//============================ +// Sign Activation +//============================ + +Matrix* sign_activation(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Sign activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = 1.0; + } else if (m->matrix_data[i] < 0) { + result->matrix_data[i] = -1.0; + } else { + result->matrix_data[i] = 0.0; + } + } + return result; +} + +Matrix* sign_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Sign_prime activation to a %dx%d matrix.", m->rows, m->cols); + // The derivative of the sign function is 0 everywhere except at 0, where it is undefined. + // For backpropagation, the derivative is commonly approximated as 0. + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + result->matrix_data[i] = 0.0; + } + return result; +} + +//============================ +// Identity Activation +//============================ + +Matrix* identity_activation(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Identity activation to a %dx%d matrix.", m->rows, m->cols); + // Simply return a copy of the input matrix as the output is identical + return copy_matrix(m); +} + +Matrix* identity_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Identity_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + result->matrix_data[i] = 1.0; + } + return result; +} + +//============================ +// Hard Tanh Activation +//============================ + +Matrix* hard_tanh(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Hard Tanh activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 1.0) { + result->matrix_data[i] = 1.0; + } else if (m->matrix_data[i] < -1.0) { + result->matrix_data[i] = -1.0; + } else { + result->matrix_data[i] = m->matrix_data[i]; + } + } + return result; +} + +Matrix* hard_tanh_prime(Matrix* m) { + ASSERT(m != NULL, "Input matrix is NULL."); + LOG_INFO("Applying Hard Tanh_prime activation to a %dx%d matrix.", m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > -1.0 && m->matrix_data[i] < 1.0) { + result->matrix_data[i] = 1.0; + } else { + result->matrix_data[i] = 0.0; + } + } + return result; +} From 13084dae0aadc9cc433c0b948a2f60bd4c4438da Mon Sep 17 00:00:00 2001 From: Shreyash Date: Sun, 31 Aug 2025 22:51:53 +0530 Subject: [PATCH 2/4] feat: add user defined alpha function definitions for leaky ReLU --- nn/include/activation.h | 4 +++ nn/src/activation/activation.c | 48 ++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/nn/include/activation.h b/nn/include/activation.h index 919ca2a..76ff739 100644 --- a/nn/include/activation.h +++ b/nn/include/activation.h @@ -21,6 +21,10 @@ Matrix* tanh_prime(Matrix* m); Matrix* leaky_relu(Matrix* m); Matrix* leaky_relu_prime(Matrix* m); +// Allows the user to specify a custom alpha value +Matrix* leaky_relu_with_alpha(Matrix* m, double alpha); +Matrix* leaky_relu_prime_with_alpha(Matrix* m, double alpha); + Matrix* sign_activation(Matrix* m); Matrix* sign_prime(Matrix* m); diff --git a/nn/src/activation/activation.c b/nn/src/activation/activation.c index 3d2d34e..ac1e974 100644 --- a/nn/src/activation/activation.c +++ b/nn/src/activation/activation.c @@ -105,6 +105,8 @@ Matrix* tanh_prime(Matrix* m) { // Leaky ReLU Activation //============================ +// Without explicit definition, implemented in the functions below this +// This will assume the alpha Matrix* leaky_relu(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); LOG_INFO("Applying Leaky ReLU activation to a %dx%d matrix.", m->rows, m->cols); @@ -115,7 +117,6 @@ Matrix* leaky_relu(Matrix* m) { if (m->matrix_data[i] > 0) { result->matrix_data[i] = m->matrix_data[i]; } else { - // Common practice to use a small alpha, like 0.01 result->matrix_data[i] = 0.01 * m->matrix_data[i]; } } @@ -138,6 +139,50 @@ Matrix* leaky_relu_prime(Matrix* m) { return result; } +// For when users may require more explicit defintions of alpha +Matrix* leaky_relu_with_alpha(Matrix* m, double alpha) { + ASSERT(m != NULL, "Input matrix is NULL."); + // If I converted a non acceptable value of alpha into 0.01, it would bring in debug troubles. + ASSERT(alpha >= 0.0, "Alpha value must be non-negative."); + + LOG_INFO( + "Applying Leaky ReLU with alpha=%.2f activation function to a %dx%d " + "matrix.", + alpha, m->rows, m->cols); + + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = m->matrix_data[i]; + } else { + result->matrix_data[i] = alpha * m->matrix_data[i]; + } + } + + return result; +} + +Matrix* leaky_relu_prime_with_alpha(Matrix* m, double alpha) { + ASSERT(m != NULL, "Input matrix for leaky_relu_prime is NULL."); + ASSERT(alpha >= 0.0, "Alpha value must be non-negative."); + LOG_INFO( + "Applying Leaky ReLU with alpha=%.2f derivative to a %dx%d matrix.", + alpha, m->rows, m->cols); + Matrix* result = create_matrix(m->rows, m->cols); + int total_elements = m->rows * m->cols; + + for (int i = 0; i < total_elements; i++) { + if (m->matrix_data[i] > 0) { + result->matrix_data[i] = 1.0; + } else { + result->matrix_data[i] = alpha; + } + } + + return result; +} + //============================ // Sign Activation //============================ @@ -180,7 +225,6 @@ Matrix* sign_prime(Matrix* m) { Matrix* identity_activation(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); LOG_INFO("Applying Identity activation to a %dx%d matrix.", m->rows, m->cols); - // Simply return a copy of the input matrix as the output is identical return copy_matrix(m); } From 81793c9ac9309ee9ec886c0426bb00798b76e185 Mon Sep 17 00:00:00 2001 From: Shreyash Date: Sun, 31 Aug 2025 22:57:52 +0530 Subject: [PATCH 3/4] style: rename alpha to leak_parameter for better naming clarity --- nn/include/activation.h | 4 ++-- nn/src/activation/activation.c | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/nn/include/activation.h b/nn/include/activation.h index 76ff739..b55e27d 100644 --- a/nn/include/activation.h +++ b/nn/include/activation.h @@ -22,8 +22,8 @@ Matrix* leaky_relu(Matrix* m); Matrix* leaky_relu_prime(Matrix* m); // Allows the user to specify a custom alpha value -Matrix* leaky_relu_with_alpha(Matrix* m, double alpha); -Matrix* leaky_relu_prime_with_alpha(Matrix* m, double alpha); +Matrix* leaky_relu_with_alpha(Matrix* m, double leak_parameter); +Matrix* leaky_relu_prime_with_alpha(Matrix* m, double leak_parameter); Matrix* sign_activation(Matrix* m); Matrix* sign_prime(Matrix* m); diff --git a/nn/src/activation/activation.c b/nn/src/activation/activation.c index ac1e974..0b67753 100644 --- a/nn/src/activation/activation.c +++ b/nn/src/activation/activation.c @@ -140,15 +140,15 @@ Matrix* leaky_relu_prime(Matrix* m) { } // For when users may require more explicit defintions of alpha -Matrix* leaky_relu_with_alpha(Matrix* m, double alpha) { +Matrix* leaky_relu_with_alpha(Matrix* m, double leak_parameter) { ASSERT(m != NULL, "Input matrix is NULL."); // If I converted a non acceptable value of alpha into 0.01, it would bring in debug troubles. - ASSERT(alpha >= 0.0, "Alpha value must be non-negative."); + ASSERT(leak_parameter >= 0.0, "Alpha value must be non-negative."); LOG_INFO( - "Applying Leaky ReLU with alpha=%.2f activation function to a %dx%d " + "Applying Leaky ReLU with leak_parameter=%.2f activation function to a %dx%d " "matrix.", - alpha, m->rows, m->cols); + leak_parameter, m->rows, m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -156,19 +156,19 @@ Matrix* leaky_relu_with_alpha(Matrix* m, double alpha) { if (m->matrix_data[i] > 0) { result->matrix_data[i] = m->matrix_data[i]; } else { - result->matrix_data[i] = alpha * m->matrix_data[i]; + result->matrix_data[i] = leak_parameter * m->matrix_data[i]; } } return result; } -Matrix* leaky_relu_prime_with_alpha(Matrix* m, double alpha) { +Matrix* leaky_relu_prime_with_alpha(Matrix* m, double leak_parameter) { ASSERT(m != NULL, "Input matrix for leaky_relu_prime is NULL."); - ASSERT(alpha >= 0.0, "Alpha value must be non-negative."); + ASSERT(leak_parameter >= 0.0, "Alpha value must be non-negative."); LOG_INFO( "Applying Leaky ReLU with alpha=%.2f derivative to a %dx%d matrix.", - alpha, m->rows, m->cols); + leak_parameter, m->rows, m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -176,7 +176,7 @@ Matrix* leaky_relu_prime_with_alpha(Matrix* m, double alpha) { if (m->matrix_data[i] > 0) { result->matrix_data[i] = 1.0; } else { - result->matrix_data[i] = alpha; + result->matrix_data[i] = leak_parameter; } } From 3fe69ac5628041c48d763ff09df76ac4663c12e7 Mon Sep 17 00:00:00 2001 From: Shreyash Date: Sun, 31 Aug 2025 23:03:14 +0530 Subject: [PATCH 4/4] style: format code --- nn/src/activation/activation.c | 50 +++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/nn/src/activation/activation.c b/nn/src/activation/activation.c index 0b67753..32f32e9 100644 --- a/nn/src/activation/activation.c +++ b/nn/src/activation/activation.c @@ -1,9 +1,10 @@ +#include "activation.h" + #include #include #include #include -#include "activation.h" #include "linalg.h" #include "utils.h" @@ -25,8 +26,9 @@ Matrix* sigmoid(Matrix* m) { Matrix* sigmoid_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying sigmoid_prime activation to a %dx%d matrix.", m->rows, m->cols); - + LOG_INFO("Applying sigmoid_prime activation to a %dx%d matrix.", m->rows, + m->cols); + Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; for (int i = 0; i < total_elements; i++) { @@ -58,7 +60,8 @@ Matrix* relu(Matrix* m) { Matrix* relu_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying ReLU_prime activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying ReLU_prime activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -90,7 +93,8 @@ Matrix* tanh_activation(Matrix* m) { Matrix* tanh_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Tanh_prime activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying Tanh_prime activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -109,7 +113,8 @@ Matrix* tanh_prime(Matrix* m) { // This will assume the alpha Matrix* leaky_relu(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Leaky ReLU activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying Leaky ReLU activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -125,7 +130,8 @@ Matrix* leaky_relu(Matrix* m) { Matrix* leaky_relu_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Leaky ReLU_prime activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying Leaky ReLU_prime activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -142,11 +148,13 @@ Matrix* leaky_relu_prime(Matrix* m) { // For when users may require more explicit defintions of alpha Matrix* leaky_relu_with_alpha(Matrix* m, double leak_parameter) { ASSERT(m != NULL, "Input matrix is NULL."); - // If I converted a non acceptable value of alpha into 0.01, it would bring in debug troubles. + // If I converted a non acceptable value of alpha into 0.01, it would bring in + // debug troubles. ASSERT(leak_parameter >= 0.0, "Alpha value must be non-negative."); LOG_INFO( - "Applying Leaky ReLU with leak_parameter=%.2f activation function to a %dx%d " + "Applying Leaky ReLU with leak_parameter=%.2f activation function to a " + "%dx%d " "matrix.", leak_parameter, m->rows, m->cols); @@ -166,9 +174,8 @@ Matrix* leaky_relu_with_alpha(Matrix* m, double leak_parameter) { Matrix* leaky_relu_prime_with_alpha(Matrix* m, double leak_parameter) { ASSERT(m != NULL, "Input matrix for leaky_relu_prime is NULL."); ASSERT(leak_parameter >= 0.0, "Alpha value must be non-negative."); - LOG_INFO( - "Applying Leaky ReLU with alpha=%.2f derivative to a %dx%d matrix.", - leak_parameter, m->rows, m->cols); + LOG_INFO("Applying Leaky ReLU with alpha=%.2f derivative to a %dx%d matrix.", + leak_parameter, m->rows, m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -207,9 +214,11 @@ Matrix* sign_activation(Matrix* m) { Matrix* sign_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Sign_prime activation to a %dx%d matrix.", m->rows, m->cols); - // The derivative of the sign function is 0 everywhere except at 0, where it is undefined. - // For backpropagation, the derivative is commonly approximated as 0. + LOG_INFO("Applying Sign_prime activation to a %dx%d matrix.", m->rows, + m->cols); + // The derivative of the sign function is 0 everywhere except at 0, where it + // is undefined. For backpropagation, the derivative is commonly approximated + // as 0. Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; for (int i = 0; i < total_elements; i++) { @@ -230,7 +239,8 @@ Matrix* identity_activation(Matrix* m) { Matrix* identity_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Identity_prime activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying Identity_prime activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -246,7 +256,8 @@ Matrix* identity_prime(Matrix* m) { Matrix* hard_tanh(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Hard Tanh activation to a %dx%d matrix.", m->rows, m->cols); + LOG_INFO("Applying Hard Tanh activation to a %dx%d matrix.", m->rows, + m->cols); Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; @@ -264,8 +275,9 @@ Matrix* hard_tanh(Matrix* m) { Matrix* hard_tanh_prime(Matrix* m) { ASSERT(m != NULL, "Input matrix is NULL."); - LOG_INFO("Applying Hard Tanh_prime activation to a %dx%d matrix.", m->rows, m->cols); - + LOG_INFO("Applying Hard Tanh_prime activation to a %dx%d matrix.", m->rows, + m->cols); + Matrix* result = create_matrix(m->rows, m->cols); int total_elements = m->rows * m->cols; for (int i = 0; i < total_elements; i++) {