From fa1a3593392ccf012f8852f4e116177a12f21028 Mon Sep 17 00:00:00 2001 From: Shreyash Date: Tue, 16 Sep 2025 01:28:43 +0530 Subject: [PATCH 1/2] feat: add backprop library --- nn/include/backprop.h | 19 ++++ nn/src/neural_network/backprop.c | 172 +++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 nn/include/backprop.h create mode 100644 nn/src/neural_network/backprop.c diff --git a/nn/include/backprop.h b/nn/include/backprop.h new file mode 100644 index 0000000..e913a49 --- /dev/null +++ b/nn/include/backprop.h @@ -0,0 +1,19 @@ +#pragma once + +#include "loss.h" +#include "neural_network.h" + +//============================ +// Functions for Backpropagation +//============================ + +void backpropagate(NeuralNetwork* nn, const Matrix* y_true, + LossFunction loss_func, LossFunctionGrad loss_func_grad); + +// Calculates the gradient of a single layer's weights. +Matrix* calculate_weight_gradient(const Cache* cache, int layer_index, + int total_layers); + +// Calculates the gradient of a single layer's bias. +Matrix* calculate_bias_gradient(const Cache* cache, int layer_index, + int total_layers); diff --git a/nn/src/neural_network/backprop.c b/nn/src/neural_network/backprop.c new file mode 100644 index 0000000..66e0d16 --- /dev/null +++ b/nn/src/neural_network/backprop.c @@ -0,0 +1,172 @@ +#include "backprop.h" + +#include +#include + +#include "activation.h" +#include "cache.h" +#include "linalg.h" +#include "neural_network.h" +#include "utils.h" + +// Select and compute activation derivative for a layer given its pre-activation +// input z. Handles common activations and leaky ReLU with optional alpha. +static Matrix* activation_derivative_for_layer(const Layer* layer, Matrix* z) { + ASSERT(layer != NULL, "Layer cannot be NULL."); + ASSERT(z != NULL, "Pre-activation matrix z cannot be NULL."); + + if (layer->activation == sigmoid) { + return sigmoid_prime(z); + } + if (layer->activation == relu) { + return relu_prime(z); + } + if (layer->activation == tanh_activation) { + return tanh_prime(z); + } + if (layer->activation == leaky_relu) { + return leaky_relu_prime(z); + } + if (layer->activation == leaky_relu_with_alpha) { + return leaky_relu_prime_with_alpha(z, layer->leak_parameter); + } + if (layer->activation == sign_activation) { + return sign_prime(z); + } + if (layer->activation == identity_activation) { + return identity_prime(z); + } + if (layer->activation == hard_tanh) { + return hard_tanh_prime(z); + } + + // Default: identity derivative + LOG_WARN("Unknown activation function, defaulting derivative to identity."); + return identity_prime(z); +} + +void backpropagate(NeuralNetwork* nn, const Matrix* y_true, LossFunction loss_func, + LossFunctionGrad loss_func_grad) { + ASSERT(nn != NULL, "Neural Network pointer cannot be NULL."); + ASSERT(nn->cache != NULL, "Cache cannot be NULL."); + ASSERT(y_true != NULL, "Ground truth matrix cannot be NULL."); + ASSERT(loss_func_grad != NULL, "Loss gradient function cannot be NULL."); + + int last_index = nn->num_layers - 1; + + // Get y_hat from cache (activation of last layer) + char a_last_key[32]; + sprintf(a_last_key, "a_%d", last_index); + Matrix* y_hat = get_matrix(nn->cache, a_last_key); + ASSERT(y_hat != NULL, "Cached prediction (y_hat) not found."); + + // dL/da for output layer + Matrix* dL_da = loss_func_grad(y_hat, y_true); + ASSERT(dL_da != NULL, "Loss gradient returned NULL."); + + // delta for output layer: dL/dz = dL/da .* a'(z) + char z_last_key[32]; + sprintf(z_last_key, "z_%d", last_index); + Matrix* z_last = get_matrix(nn->cache, z_last_key); + ASSERT(z_last != NULL, "Cached z for last layer not found."); + + Matrix* act_prime_last = activation_derivative_for_layer(nn->layers[last_index], z_last); + Matrix* delta_last = multiply_matrix(dL_da, act_prime_last); + ASSERT(delta_last != NULL, "Failed to compute delta for last layer."); + + char delta_last_key[32]; + sprintf(delta_last_key, "delta_%d", last_index); + put_matrix(nn->cache, delta_last_key, delta_last); + + // Clean up temporaries for last layer + free_matrix(y_hat); + free_matrix(dL_da); + free_matrix(z_last); + free_matrix(act_prime_last); + free_matrix(delta_last); + + // Backpropagate through hidden layers + for (int i = last_index - 1; i >= 0; i--) { + // delta_{i} = (delta_{i+1} dot W_{i+1}^T) .* a'_i(z_i) + char delta_next_key[32]; + sprintf(delta_next_key, "delta_%d", i + 1); + Matrix* delta_next = get_matrix(nn->cache, delta_next_key); + ASSERT(delta_next != NULL, "Cached delta for next layer not found."); + + Matrix* W_next = nn->layers[i + 1]->weights; + ASSERT(W_next != NULL, "Weights for next layer cannot be NULL."); + + Matrix* W_next_T = transpose_matrix(W_next); + Matrix* propagated = dot_matrix(delta_next, W_next_T); + + char z_key[32]; + sprintf(z_key, "z_%d", i); + Matrix* z_i = get_matrix(nn->cache, z_key); + ASSERT(z_i != NULL, "Cached z for layer not found."); + Matrix* act_prime_i = activation_derivative_for_layer(nn->layers[i], z_i); + + Matrix* delta_i = multiply_matrix(propagated, act_prime_i); + ASSERT(delta_i != NULL, "Failed to compute delta for layer."); + + char delta_i_key[32]; + sprintf(delta_i_key, "delta_%d", i); + put_matrix(nn->cache, delta_i_key, delta_i); + + // Clean up + free_matrix(delta_next); + free_matrix(W_next_T); + free_matrix(propagated); + free_matrix(z_i); + free_matrix(act_prime_i); + free_matrix(delta_i); + } +} + +Matrix* calculate_weight_gradient(const Cache* cache, int layer_index, int total_layers) { + ASSERT(cache != NULL, "Cache cannot be NULL."); + ASSERT(layer_index >= 0 && layer_index < total_layers, + "layer_index out of bounds."); + + // Get activation of previous layer (or input) + Matrix* a_prev = NULL; + if (layer_index == 0) { + a_prev = get_matrix((Cache*)cache, "input"); + } else { + char a_prev_key[32]; + sprintf(a_prev_key, "a_%d", layer_index - 1); + a_prev = get_matrix((Cache*)cache, a_prev_key); + } + ASSERT(a_prev != NULL, "Cached previous activation/input not found."); + + // Get delta for current layer + char delta_key[32]; + sprintf(delta_key, "delta_%d", layer_index); + Matrix* delta_i = get_matrix((Cache*)cache, delta_key); + ASSERT(delta_i != NULL, "Cached delta for layer not found."); + + Matrix* a_prev_T = transpose_matrix(a_prev); + Matrix* grad_W = dot_matrix(a_prev_T, delta_i); + + // Clean up + free_matrix(a_prev); + free_matrix(delta_i); + free_matrix(a_prev_T); + + return grad_W; +} + +Matrix* calculate_bias_gradient(const Cache* cache, int layer_index, int total_layers) { + ASSERT(cache != NULL, "Cache cannot be NULL."); + ASSERT(layer_index >= 0 && layer_index < total_layers, + "layer_index out of bounds."); + + char delta_key[32]; + sprintf(delta_key, "delta_%d", layer_index); + Matrix* delta_i = get_matrix((Cache*)cache, delta_key); + ASSERT(delta_i != NULL, "Cached delta for layer not found."); + + // For single-sample case, bias gradient equals delta + return delta_i; // already a deep copy from cache +} + + From 8172811d3b2a74664a93fbc27c7672bb8cb91b7c Mon Sep 17 00:00:00 2001 From: Shreyash Date: Tue, 16 Sep 2025 01:33:49 +0530 Subject: [PATCH 2/2] style: fix formatting --- nn/src/neural_network/backprop.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nn/src/neural_network/backprop.c b/nn/src/neural_network/backprop.c index 66e0d16..cc767a0 100644 --- a/nn/src/neural_network/backprop.c +++ b/nn/src/neural_network/backprop.c @@ -45,8 +45,8 @@ static Matrix* activation_derivative_for_layer(const Layer* layer, Matrix* z) { return identity_prime(z); } -void backpropagate(NeuralNetwork* nn, const Matrix* y_true, LossFunction loss_func, - LossFunctionGrad loss_func_grad) { +void backpropagate(NeuralNetwork* nn, const Matrix* y_true, + LossFunction loss_func, LossFunctionGrad loss_func_grad) { ASSERT(nn != NULL, "Neural Network pointer cannot be NULL."); ASSERT(nn->cache != NULL, "Cache cannot be NULL."); ASSERT(y_true != NULL, "Ground truth matrix cannot be NULL."); @@ -70,7 +70,8 @@ void backpropagate(NeuralNetwork* nn, const Matrix* y_true, LossFunction loss_fu Matrix* z_last = get_matrix(nn->cache, z_last_key); ASSERT(z_last != NULL, "Cached z for last layer not found."); - Matrix* act_prime_last = activation_derivative_for_layer(nn->layers[last_index], z_last); + Matrix* act_prime_last = + activation_derivative_for_layer(nn->layers[last_index], z_last); Matrix* delta_last = multiply_matrix(dL_da, act_prime_last); ASSERT(delta_last != NULL, "Failed to compute delta for last layer."); @@ -122,7 +123,8 @@ void backpropagate(NeuralNetwork* nn, const Matrix* y_true, LossFunction loss_fu } } -Matrix* calculate_weight_gradient(const Cache* cache, int layer_index, int total_layers) { +Matrix* calculate_weight_gradient(const Cache* cache, int layer_index, + int total_layers) { ASSERT(cache != NULL, "Cache cannot be NULL."); ASSERT(layer_index >= 0 && layer_index < total_layers, "layer_index out of bounds."); @@ -155,7 +157,8 @@ Matrix* calculate_weight_gradient(const Cache* cache, int layer_index, int total return grad_W; } -Matrix* calculate_bias_gradient(const Cache* cache, int layer_index, int total_layers) { +Matrix* calculate_bias_gradient(const Cache* cache, int layer_index, + int total_layers) { ASSERT(cache != NULL, "Cache cannot be NULL."); ASSERT(layer_index >= 0 && layer_index < total_layers, "layer_index out of bounds."); @@ -168,5 +171,3 @@ Matrix* calculate_bias_gradient(const Cache* cache, int layer_index, int total_l // For single-sample case, bias gradient equals delta return delta_i; // already a deep copy from cache } - -