diff --git a/CMakeLists.txt b/CMakeLists.txt index cf8540e..fd315b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,6 +19,9 @@ add_library(nir_c_runtime src/fixed_event.c src/train.c src/stdp_graph.c + src/init.c + src/schedule.c + src/loss.c src/rng.c src/encode.c src/data.c diff --git a/include/nir_c_runtime/init.h b/include/nir_c_runtime/init.h new file mode 100644 index 0000000..8513efe --- /dev/null +++ b/include/nir_c_runtime/init.h @@ -0,0 +1,77 @@ +/* Weight initializers for from-scratch C training (#45). + * + * Every weight in a C-loaded graph otherwise comes from a + * Python-exported graph JSON; these fill a fresh buffer so a + * zero-Python training run can start somewhere sane, then land + * the buffer on a node via `nir_graph_set_weights`. + * + * Formulas mirror torch.nn.init exactly: + * + * uniform: U(low, high) + * xavier_uniform: U(-b, b), b = gain * sqrt(6 / (fan_in + fan_out)) + * xavier_normal: N(0, s), s = gain * sqrt(2 / (fan_in + fan_out)) + * he_uniform: U(-b, b), b = gain * sqrt(3 / fan_in) + * he_normal: N(0, s), s = gain / sqrt(fan_in) + * + * Fan conventions (torch `_calculate_fan_in_and_fan_out`, which + * `nir_init_fans` implements): a Linear/Affine weight laid out + * [out_features, in_features] has fan_in = in_features and + * fan_out = out_features; a convolution weight laid out + * [out_channels, in_channels, *kernel] multiplies both by the + * kernel's receptive-field size (kh * kw for Conv2d, k for + * Conv1d). He follows torch's kaiming defaults -- mode 'fan_in' + * (only fan_in enters) with an explicit gain, so ReLU nets pass + * sqrt(2) (torch `calculate_gain('relu')`, equal to leaky_relu + * with a = 0) and Xavier nets pass 1. Use `nir_init_fans` on the + * node's weight shape rather than hand-counting. + * + * All draws come from `nir_rng_t`, so a seeded stream replays + * bit-identically. Uniform draws consume one u64 each; normal + * draws use Box-Muller pairs (two u64 per pair, the spare + * discarded when `count` is odd), so the stream position after + * a call depends on `count` -- reseed to replay a prefix. + */ +#ifndef NIR_C_RUNTIME_INIT_H +#define NIR_C_RUNTIME_INIT_H + +#include + +#include "nir_c_runtime/rng.h" + +/* Torch fan-in/fan-out for a weight shape: shape[0] is output + * maps, shape[1] input maps, shape[2..] the kernel. Needs + * ndim >= 2, no zero extents, and products that fit size_t. */ +int nir_init_fans( + const size_t *shape, size_t ndim, size_t *fan_in, + size_t *fan_out +); + +/* Fill `out` with U(low, high). Needs low < high, both finite. */ +int nir_init_uniform( + nir_rng_t *rng, float *out, size_t count, float low, + float high +); + +/* Xavier/Glorot initializers. Needs both fans nonzero and a + * finite gain >= 0 (gain 0 yields all zeros). */ +int nir_init_xavier_uniform( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + size_t fan_out, float gain +); +int nir_init_xavier_normal( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + size_t fan_out, float gain +); + +/* He initializers (torch kaiming_* with mode 'fan_in'). Needs + * fan_in nonzero and a finite gain >= 0. */ +int nir_init_he_uniform( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + float gain +); +int nir_init_he_normal( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + float gain +); + +#endif /* NIR_C_RUNTIME_INIT_H */ diff --git a/include/nir_c_runtime/loss.h b/include/nir_c_runtime/loss.h new file mode 100644 index 0000000..b727896 --- /dev/null +++ b/include/nir_c_runtime/loss.h @@ -0,0 +1,56 @@ +/* Loss functions beyond single-sample cross-entropy (#47). + * + * `nir_cross_entropy` (train.h) stays the classifier loss for + * logits; these cover the other two common SNN recipes: + * + * nir_mse_loss: mean-squared error with 'mean' reduction + * (loss = mean((pred - target)^2), grad = 2(pred - target)/n), + * mirroring torch F.mse_loss on one sample. Serves + * readout/membrane regression -- predicting continuous + * targets such as a decoded intensity or a target membrane + * trace -- where classification-style softmax is wrong. + * + * nir_spike_count_loss: softmax cross-entropy over the + * per-class spike-count vector, mirroring torch + * F.cross_entropy(counts, target) on one sample. This is the + * decided spike-count form (cross-entropy over counts, not + * MSE on counts): it matches the snnTorch recipe of summing + * output spikes over the window and classifying the totals, + * and spikeforge's own averaged-readout-plus-cross_entropy + * training path is the same family. The caller accumulates + * the T-step window into `counts` (a plain sum; each step's + * spikes are 0/1, so counts are non-negative by + * construction) and backpropagates through the accumulation + * itself -- every timestep's spikes share their class's + * count gradient -- until the BPTT tape (#39) owns that. + * + * Both keep the single-sample-first convention: 1-D vectors, + * mean reduction for MSE, batch reduction left to the + * mini-batch issue (#42). + */ +#ifndef NIR_C_RUNTIME_LOSS_H +#define NIR_C_RUNTIME_LOSS_H + +#include + +#include "nir_c_runtime/tensor.h" + +/* MSE on one sample. `pred`/`target` must be 1-D with equal + * nonzero size and finite data; `grad` needs capacity for the + * vector and follows the cross-entropy aliasing rule (exact + * aliasing with either input is fine, partial overlap fails). */ +int nir_mse_loss( + const nir_tensor_t *pred, const nir_tensor_t *target, + float *loss, nir_tensor_t *grad +); + +/* Softmax cross-entropy over spike counts: loss = + * -log(softmax(counts)[target]), grad = softmax(counts) with + * grad[target] -= 1. Same shape/aliasing rules as + * `nir_cross_entropy`. */ +int nir_spike_count_loss( + const nir_tensor_t *counts, size_t target, float *loss, + nir_tensor_t *grad +); + +#endif /* NIR_C_RUNTIME_LOSS_H */ diff --git a/include/nir_c_runtime/schedule.h b/include/nir_c_runtime/schedule.h new file mode 100644 index 0000000..603c057 --- /dev/null +++ b/include/nir_c_runtime/schedule.h @@ -0,0 +1,52 @@ +/* Learning-rate schedules for the SGD/Adam optimizers (#46). + * + * Stateless functions computing lr from a step counter, applied + * by writing the optimizer's `lr` field before stepping: + * + * float lr = 0.0f; + * nir_lr_step_decay(0.1f, 0.5f, 10, meta.step, &lr); + * sgd.lr = lr; + * nir_sgd_step(&sgd, weights, grad, count); + * + * The same write works for `nir_adam_t.lr`. There is no + * scheduler struct to persist: the counter IS the checkpoint's + * `step` (completed optimizer steps, 0 for a fresh run), so a + * resumed run passes the loaded `meta.step` and continues the + * curve exactly where it stopped. + * + * Formulas mirror torch.optim.lr_scheduler with last_epoch set + * to `step` (step 0 always yields base_lr): + * + * step_decay: base_lr * gamma^(step / step_size) + * cosine: eta_min + (base_lr - eta_min) * + * (1 + cos(pi * step / t_max)) / 2 + * exponential: base_lr * gamma^step + * + * Cosine follows torch past t_max too: beyond the minimum the + * curve rises again rather than clamping, so callers that want + * a floor should stop the run (or the schedule) at t_max. + */ +#ifndef NIR_C_RUNTIME_SCHEDULE_H +#define NIR_C_RUNTIME_SCHEDULE_H + +/* Step decay (torch StepLR). Needs base_lr and gamma finite + * and >= 0, step_size >= 1. */ +int nir_lr_step_decay( + float base_lr, float gamma, unsigned long step_size, + unsigned long step, float *lr +); + +/* Cosine annealing (torch CosineAnnealingLR). Needs base_lr + * and eta_min finite and >= 0, t_max >= 1. */ +int nir_lr_cosine( + float base_lr, float eta_min, unsigned long t_max, + unsigned long step, float *lr +); + +/* Exponential decay (torch ExponentialLR). Needs base_lr and + * gamma finite and >= 0. */ +int nir_lr_exponential( + float base_lr, float gamma, unsigned long step, float *lr +); + +#endif /* NIR_C_RUNTIME_SCHEDULE_H */ diff --git a/src/init.c b/src/init.c new file mode 100644 index 0000000..b77b431 --- /dev/null +++ b/src/init.c @@ -0,0 +1,137 @@ +#include "nir_c_runtime/init.h" + +#include +#include + +static int field_size( + const size_t *shape, size_t ndim, size_t *field +) { + size_t f = 1; + for (size_t i = 2; i < ndim; i++) { + if (shape[i] == 0 || f > SIZE_MAX / shape[i]) { + return 0; + } + f *= shape[i]; + } + *field = f; + return 1; +} + +int nir_init_fans( + const size_t *shape, size_t ndim, size_t *fan_in, + size_t *fan_out +) { + size_t field = 0; + if (shape == NULL || ndim < 2 || fan_in == NULL || + fan_out == NULL || shape[0] == 0 || shape[1] == 0 || + !field_size(shape, ndim, &field) || + shape[1] > SIZE_MAX / field || + shape[0] > SIZE_MAX / field) { + return 0; + } + *fan_in = shape[1] * field; + *fan_out = shape[0] * field; + return 1; +} + +static int check_fill( + const nir_rng_t *rng, const float *out, size_t count +) { + return rng != NULL && out != NULL && count > 0; +} + +int nir_init_uniform( + nir_rng_t *rng, float *out, size_t count, float low, + float high +) { + if (!check_fill(rng, out, count) || !isfinite(low) || + !isfinite(high) || !(low < high)) { + return 0; + } + double span = (double)high - (double)low; + for (size_t i = 0; i < count; i++) { + double u = nir_rng_next_f64(rng); + out[i] = (float)((double)low + span * u); + } + return 1; +} + +/* One standard-normal draw via Box-Muller. u1 == 0 would feed + * log(0), so resample it; the stream stays deterministic. */ +static double gaussian(nir_rng_t *rng) { + double u1 = 0.0; + do { + u1 = nir_rng_next_f64(rng); + } while (u1 == 0.0); + double u2 = nir_rng_next_f64(rng); + double r = sqrt(-2.0 * log(u1)); + return r * cos(2.0 * acos(-1.0) * u2); +} + +static int check_gain(float gain) { + return isfinite(gain) && gain >= 0.0f; +} + +int nir_init_xavier_uniform( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + size_t fan_out, float gain +) { + if (!check_fill(rng, out, count) || fan_in == 0 || + fan_out == 0 || !check_gain(gain)) { + return 0; + } + double total = (double)fan_in + (double)fan_out; + double bound = (double)gain * sqrt(6.0 / total); + for (size_t i = 0; i < count; i++) { + double u = nir_rng_next_f64(rng); + out[i] = (float)(-bound + 2.0 * bound * u); + } + return 1; +} + +int nir_init_xavier_normal( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + size_t fan_out, float gain +) { + if (!check_fill(rng, out, count) || fan_in == 0 || + fan_out == 0 || !check_gain(gain)) { + return 0; + } + double total = (double)fan_in + (double)fan_out; + double std = (double)gain * sqrt(2.0 / total); + for (size_t i = 0; i < count; i++) { + out[i] = (float)(std * gaussian(rng)); + } + return 1; +} + +int nir_init_he_uniform( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + float gain +) { + if (!check_fill(rng, out, count) || fan_in == 0 || + !check_gain(gain)) { + return 0; + } + double bound = (double)gain * sqrt(3.0 / (double)fan_in); + for (size_t i = 0; i < count; i++) { + double u = nir_rng_next_f64(rng); + out[i] = (float)(-bound + 2.0 * bound * u); + } + return 1; +} + +int nir_init_he_normal( + nir_rng_t *rng, float *out, size_t count, size_t fan_in, + float gain +) { + if (!check_fill(rng, out, count) || fan_in == 0 || + !check_gain(gain)) { + return 0; + } + double std = (double)gain / sqrt((double)fan_in); + for (size_t i = 0; i < count; i++) { + out[i] = (float)(std * gaussian(rng)); + } + return 1; +} diff --git a/src/loss.c b/src/loss.c new file mode 100644 index 0000000..62d7f23 --- /dev/null +++ b/src/loss.c @@ -0,0 +1,119 @@ +#include "nir_c_runtime/loss.h" + +#include +#include + +/* Exact aliasing is elementwise-safe; anything else that + * overlaps is rejected, matching nir_cross_entropy. */ +static int check_alias( + const float *input, const float *other, size_t n +) { + if (input == other) return 1; + uintptr_t a = (uintptr_t)input; + uintptr_t b = (uintptr_t)other; + size_t bytes = n * sizeof(float); + return !(a < b + bytes && b < a + bytes); +} + +static int check_vector( + const nir_tensor_t *t, const nir_tensor_t *grad +) { + return t != NULL && t->data != NULL && t->ndim == 1 && + t->size > 0 && t->capacity >= t->size && + grad != NULL && grad->data != NULL && + grad->capacity >= t->size; +} + +static int check_finite(const float *data, size_t n) { + for (size_t i = 0; i < n; i++) { + if (!isfinite(data[i])) return 0; + } + return 1; +} + +static void shape_like(nir_tensor_t *grad, size_t n) { + grad->ndim = 1; + grad->shape[0] = n; + grad->shape[1] = 0; + grad->shape[2] = 0; + grad->size = n; +} + +static int check_mse( + const nir_tensor_t *pred, const nir_tensor_t *target, + const float *loss, const nir_tensor_t *grad +) { + return check_vector(pred, grad) && target != NULL && + target->data != NULL && target->ndim == 1 && + target->size == pred->size && + target->capacity >= target->size && loss != NULL && + check_finite(pred->data, pred->size) && + check_finite(target->data, target->size) && + check_alias(pred->data, grad->data, pred->size) && + check_alias(target->data, grad->data, pred->size); +} + +int nir_mse_loss( + const nir_tensor_t *pred, const nir_tensor_t *target, + float *loss, nir_tensor_t *grad +) { + if (!check_mse(pred, target, loss, grad)) return 0; + double total = 0.0; + for (size_t i = 0; i < pred->size; i++) { + double d = (double)pred->data[i] - (double)target->data[i]; + total += d * d; + } + shape_like(grad, pred->size); + double n = (double)pred->size; + for (size_t i = 0; i < pred->size; i++) { + double d = (double)pred->data[i] - (double)target->data[i]; + grad->data[i] = (float)(2.0 * d / n); + } + *loss = (float)(total / n); + return 1; +} + +static double softmax_total( + const nir_tensor_t *counts, float *peak +) { + float p = counts->data[0]; + for (size_t i = 1; i < counts->size; i++) { + if (counts->data[i] > p) p = counts->data[i]; + } + *peak = p; + double total = 0.0; + for (size_t i = 0; i < counts->size; i++) { + total += exp((double)counts->data[i] - (double)p); + } + return total; +} + +static void softmax_into( + const nir_tensor_t *counts, size_t target, nir_tensor_t *grad, + float *loss +) { + float peak = 0.0f; + double total = softmax_total(counts, &peak); + shape_like(grad, counts->size); + for (size_t i = 0; i < counts->size; i++) { + double p = exp((double)counts->data[i] - (double)peak); + grad->data[i] = (float)(p / total - + (i == target ? 1.0 : 0.0)); + } + *loss = (float)-log( + exp((double)counts->data[target] - (double)peak) / total + ); +} + +int nir_spike_count_loss( + const nir_tensor_t *counts, size_t target, float *loss, + nir_tensor_t *grad +) { + if (!check_vector(counts, grad) || target >= counts->size || + loss == NULL || !check_finite(counts->data, counts->size) || + !check_alias(counts->data, grad->data, counts->size)) { + return 0; + } + softmax_into(counts, target, grad, loss); + return 1; +} diff --git a/src/schedule.c b/src/schedule.c new file mode 100644 index 0000000..6ccc61e --- /dev/null +++ b/src/schedule.c @@ -0,0 +1,51 @@ +#include "nir_c_runtime/schedule.h" + +#include +#include + +static int check_base(float base_lr, const float *lr) { + return lr != NULL && isfinite(base_lr) && base_lr >= 0.0f; +} + +int nir_lr_step_decay( + float base_lr, float gamma, unsigned long step_size, + unsigned long step, float *lr +) { + if (!check_base(base_lr, lr) || !isfinite(gamma) || + gamma < 0.0f || step_size == 0) { + return 0; + } + double factor = pow( + (double)gamma, (double)(step / step_size) + ); + *lr = (float)((double)base_lr * factor); + return 1; +} + +int nir_lr_cosine( + float base_lr, float eta_min, unsigned long t_max, + unsigned long step, float *lr +) { + if (!check_base(base_lr, lr) || !isfinite(eta_min) || + eta_min < 0.0f || t_max == 0) { + return 0; + } + double pi = acos(-1.0); + double half = (1.0 + cos(pi * (double)step / (double)t_max)); + half /= 2.0; + *lr = (float)((double)eta_min + + ((double)base_lr - (double)eta_min) * half); + return 1; +} + +int nir_lr_exponential( + float base_lr, float gamma, unsigned long step, float *lr +) { + if (!check_base(base_lr, lr) || !isfinite(gamma) || + gamma < 0.0f) { + return 0; + } + *lr = (float)((double)base_lr * pow((double)gamma, + (double)step)); + return 1; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0cfd331..e91f617 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,10 @@ nir_add_test(test_tensor) nir_add_test(test_error_format) nir_add_test(test_train) nir_add_test(test_stdp_graph) +nir_add_test(test_init) +nir_add_test(test_init_graph) +nir_add_test(test_schedule) +nir_add_test(test_loss) nir_add_test(test_rng) nir_add_test(test_encode) nir_add_test(test_data) diff --git a/tests/test_init.c b/tests/test_init.c new file mode 100644 index 0000000..8999d52 --- /dev/null +++ b/tests/test_init.c @@ -0,0 +1,190 @@ +/* Weight initializers: torch formulas, fan conventions, + * seeded reproducibility, set_weights + forward (#45). */ +#include +#include +#include +#include + +#include "nir_c_runtime/init.h" +#include "test_util.h" + +#define STAT_N 20000 + +static void mean_var( + const float *w, size_t n, double *mean, double *var +) { + double m = 0.0; + for (size_t i = 0; i < n; i++) m += (double)w[i]; + m /= (double)n; + double v = 0.0; + for (size_t i = 0; i < n; i++) { + double d = (double)w[i] - m; + v += d * d; + } + *mean = m; + *var = v / (double)n; +} + +static float *fresh_buffer(void) { + float *w = (float *)malloc(STAT_N * sizeof(float)); + CHECK(w != NULL); + return w; +} + +static void check_within( + const float *a, size_t n, double bound +) { + for (size_t i = 0; i < n; i++) { + CHECK(fabs(a[i]) <= bound); + } +} + +static void test_fans(void) { + size_t in = 0; + size_t out = 0; + size_t linear[2] = {3, 4}; + CHECK(nir_init_fans(linear, 2, &in, &out)); + CHECK(in == 4 && out == 3); + size_t conv[4] = {16, 3, 3, 3}; + CHECK(nir_init_fans(conv, 4, &in, &out)); + CHECK(in == 27 && out == 144); + size_t conv1[3] = {8, 4, 5}; + CHECK(nir_init_fans(conv1, 3, &in, &out)); + CHECK(in == 20 && out == 40); +} + +static void test_fan_errors(void) { + size_t in = 0; + size_t out = 0; + size_t shape[2] = {3, 4}; + size_t zero[2] = {3, 0}; + size_t big[3] = {SIZE_MAX, 2, SIZE_MAX}; + CHECK(!nir_init_fans(shape, 1, &in, &out)); + CHECK(!nir_init_fans(NULL, 2, &in, &out)); + CHECK(!nir_init_fans(shape, 2, NULL, &out)); + CHECK(!nir_init_fans(zero, 2, &in, &out)); + CHECK(!nir_init_fans(big, 3, &in, &out)); +} + +static void test_uniform(void) { + float *a = fresh_buffer(); + float *b = fresh_buffer(); + nir_rng_t rng; + nir_rng_seed(&rng, 1234); + CHECK(nir_init_uniform(&rng, a, STAT_N, -2.0f, 2.0f)); + nir_rng_seed(&rng, 1234); + CHECK(nir_init_uniform(&rng, b, STAT_N, -2.0f, 2.0f)); + CHECK(memcmp(a, b, STAT_N * sizeof(float)) == 0); + double mean = 0.0; + double var = 0.0; + mean_var(a, STAT_N, &mean, &var); + CHECK_NEAR(mean, 0.0, 0.05); + CHECK_NEAR(var, 1.3333333333333333, 0.08); + free(a); + free(b); +} + +static void test_xavier_uniform(void) { + float *a = fresh_buffer(); + float *b = fresh_buffer(); + nir_rng_t rng; + nir_rng_seed(&rng, 7); + CHECK(nir_init_xavier_uniform(&rng, a, STAT_N, 128, 64, 1.0f)); + nir_rng_seed(&rng, 7); + CHECK(nir_init_xavier_uniform(&rng, b, STAT_N, 128, 64, 1.0f)); + CHECK(memcmp(a, b, STAT_N * sizeof(float)) == 0); + check_within(a, STAT_N, sqrt(6.0 / 192.0)); + double mean = 0.0; + double var = 0.0; + mean_var(a, STAT_N, &mean, &var); + CHECK_NEAR(mean, 0.0, 0.01); + CHECK_NEAR(var, 0.010416666666666666, 0.001); + free(a); + free(b); +} + +static void test_xavier_normal(void) { + float *a = fresh_buffer(); + float *b = fresh_buffer(); + nir_rng_t rng; + nir_rng_seed(&rng, 7); + CHECK(nir_init_xavier_normal(&rng, a, STAT_N, 128, 64, 1.0f)); + nir_rng_seed(&rng, 7); + CHECK(nir_init_xavier_normal(&rng, b, STAT_N, 128, 64, 1.0f)); + CHECK(memcmp(a, b, STAT_N * sizeof(float)) == 0); + double mean = 0.0; + double var = 0.0; + mean_var(a, STAT_N, &mean, &var); + CHECK_NEAR(mean, 0.0, 0.01); + CHECK_NEAR(var, 0.010416666666666666, 0.0015); + free(a); + free(b); +} + +static void test_he_uniform(void) { + float gain = (float)sqrt(2.0); + float *a = fresh_buffer(); + float *b = fresh_buffer(); + nir_rng_t rng; + nir_rng_seed(&rng, 21); + CHECK(nir_init_he_uniform(&rng, a, STAT_N, 128, gain)); + nir_rng_seed(&rng, 21); + CHECK(nir_init_he_uniform(&rng, b, STAT_N, 128, gain)); + CHECK(memcmp(a, b, STAT_N * sizeof(float)) == 0); + double bound = (double)gain * sqrt(3.0 / 128.0); + check_within(a, STAT_N, bound); + double mean = 0.0; + double var = 0.0; + mean_var(a, STAT_N, &mean, &var); + CHECK_NEAR(mean, 0.0, 0.01); + CHECK_NEAR(var, 0.015625, 0.0015); + free(a); + free(b); +} + +static void test_he_normal(void) { + float gain = (float)sqrt(2.0); + float *a = fresh_buffer(); + float *b = fresh_buffer(); + nir_rng_t rng; + nir_rng_seed(&rng, 21); + CHECK(nir_init_he_normal(&rng, a, STAT_N, 128, gain)); + nir_rng_seed(&rng, 21); + CHECK(nir_init_he_normal(&rng, b, STAT_N, 128, gain)); + CHECK(memcmp(a, b, STAT_N * sizeof(float)) == 0); + double mean = 0.0; + double var = 0.0; + mean_var(a, STAT_N, &mean, &var); + CHECK_NEAR(mean, 0.0, 0.01); + CHECK_NEAR(var, 0.015625, 0.0015); + free(a); + free(b); +} + +static void test_init_errors(void) { + nir_rng_t rng; + nir_rng_seed(&rng, 1); + float w[8]; + CHECK(!nir_init_uniform(NULL, w, 8, 0.0f, 1.0f)); + CHECK(!nir_init_uniform(&rng, NULL, 8, 0.0f, 1.0f)); + CHECK(!nir_init_uniform(&rng, w, 0, 0.0f, 1.0f)); + CHECK(!nir_init_uniform(&rng, w, 8, 1.0f, 1.0f)); + CHECK(!nir_init_uniform(&rng, w, 8, 2.0f, 1.0f)); + CHECK(!nir_init_xavier_uniform(&rng, w, 8, 0, 4, 1.0f)); + CHECK(!nir_init_xavier_normal(&rng, w, 8, 4, 0, 1.0f)); + CHECK(!nir_init_xavier_uniform(&rng, w, 8, 4, 4, -1.0f)); + CHECK(!nir_init_he_uniform(&rng, w, 8, 0, 1.0f)); + CHECK(!nir_init_he_normal(&rng, w, 8, 4, (float)NAN)); +} + +int main(void) { + test_fans(); + test_fan_errors(); + test_uniform(); + test_xavier_uniform(); + test_xavier_normal(); + test_he_uniform(); + test_he_normal(); + test_init_errors(); + return 0; +} diff --git a/tests/test_init_graph.c b/tests/test_init_graph.c new file mode 100644 index 0000000..d0ab020 --- /dev/null +++ b/tests/test_init_graph.c @@ -0,0 +1,102 @@ +/* Initialized weights land on a graph node through + * nir_graph_set_weights and run forward cleanly (#45). */ +#include +#include +#include + +#include "nir_c_runtime/checkpoint.h" +#include "nir_c_runtime/graph.h" +#include "nir_c_runtime/init.h" +#include "nir_c_runtime/json.h" +#include "nir_c_runtime/runtime.h" +#include "test_util.h" + +static nir_graph_t load_fixture(const char *path) { + nir_graph_t graph; + memset(&graph, 0, sizeof(graph)); + char *text = read_file(path); + char *error = NULL; + json_value_t *json = json_parse(text, &error); + CHECK(error == NULL); + CHECK(nir_graph_load(json, &graph, &error)); + CHECK(error == NULL); + json_free(json); + free(text); + return graph; +} + +static void fill_frames(nir_tensor_t frames[6]) { + size_t shape[1] = {4}; + for (int t = 0; t < 6; t++) { + frames[t] = nir_tensor_alloc(shape, 1); + for (size_t i = 0; i < frames[t].size; i++) { + frames[t].data[i] = (float)(t + 1) * 0.1f; + } + } +} + +static void check_forward(nir_graph_t *graph) { + nir_runtime_t runtime; + char *error = NULL; + CHECK(nir_runtime_init(&runtime, graph, &error)); + nir_tensor_t frames[6]; + fill_frames(frames); + nir_tensor_t readout; + memset(&readout, 0, sizeof(readout)); + CHECK(nir_runtime_run(&runtime, frames, 6, &readout, &error)); + CHECK(error == NULL); + CHECK(readout.size == 2); + for (size_t i = 0; i < readout.size; i++) { + CHECK(isfinite(readout.data[i])); + } + for (int t = 0; t < 6; t++) nir_tensor_free(&frames[t]); + nir_tensor_free(&readout); + nir_runtime_free(&runtime); +} + +static void check_fc1_matches( + nir_graph_t *graph, const float *fresh, size_t wn +) { + const float *w = NULL; + const float *b = NULL; + size_t wn_back = 0; + size_t bn = 0; + CHECK(nir_graph_get_weights( + graph, "fc1", &w, &wn_back, &b, &bn + )); + CHECK(wn_back == wn); + CHECK(memcmp(w, fresh, wn * sizeof(float)) == 0); +} + +static void set_fc1_xavier(nir_graph_t *graph, size_t wn) { + size_t in = 0; + size_t out = 0; + size_t shape[2] = {3, 4}; + CHECK(nir_init_fans(shape, 2, &in, &out)); + float *fresh = (float *)malloc(wn * sizeof(float)); + CHECK(fresh != NULL); + nir_rng_t rng; + nir_rng_seed(&rng, 42); + CHECK(nir_init_xavier_uniform(&rng, fresh, wn, in, out, 1.0f)); + char *error = NULL; + CHECK(nir_graph_set_weights( + graph, "fc1", fresh, wn, NULL, 0, &error + )); + CHECK(error == NULL); + check_fc1_matches(graph, fresh, wn); + free(fresh); +} + +int main(void) { + nir_graph_t graph = load_fixture("fixtures/lif_linear_graph.json"); + const float *w = NULL; + const float *b = NULL; + size_t wn = 0; + size_t bn = 0; + CHECK(nir_graph_get_weights(&graph, "fc1", &w, &wn, &b, &bn)); + CHECK(wn == 12 && b == NULL && bn == 0); + set_fc1_xavier(&graph, wn); + check_forward(&graph); + nir_graph_free(&graph); + return 0; +} diff --git a/tests/test_loss.c b/tests/test_loss.c new file mode 100644 index 0000000..bffd097 --- /dev/null +++ b/tests/test_loss.c @@ -0,0 +1,175 @@ +/* MSE and spike-count losses: closed-form value and + * gradient, cross-entropy equivalence, error cases (#47). */ +#include +#include +#include + +#include "nir_c_runtime/loss.h" +#include "nir_c_runtime/train.h" +#include "test_util.h" + +static nir_tensor_t make_vector(const float *values, size_t n) { + size_t shape[1] = {n}; + nir_tensor_t t = nir_tensor_alloc(shape, 1); + CHECK(t.data != NULL); + memcpy(t.data, values, n * sizeof(float)); + return t; +} + +static void check_grad( + const nir_tensor_t *t, const double *want, size_t n +) { + CHECK(t->size == n); + for (size_t i = 0; i < n; i++) { + CHECK_NEAR(t->data[i], want[i], 1e-6); + } +} + +static void test_mse(void) { + float pv[] = {1.0f, 2.0f, 3.0f}; + float tv[] = {1.0f, 0.0f, 3.0f}; + nir_tensor_t pred = make_vector(pv, 3); + nir_tensor_t target = make_vector(tv, 3); + size_t shape[1] = {3}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL); + float loss = 0.0f; + CHECK(nir_mse_loss(&pred, &target, &loss, &grad)); + CHECK_NEAR(loss, 1.3333333333333333, 1e-6); + double want[] = {0.0, 1.3333333333333333, 0.0}; + check_grad(&grad, want, 3); + nir_tensor_free(&pred); + nir_tensor_free(&target); + nir_tensor_free(&grad); +} + +static void test_mse_zero(void) { + float pv[] = {1.0f, 2.0f, 3.0f}; + nir_tensor_t pred = make_vector(pv, 3); + size_t shape[1] = {3}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL); + float loss = 0.0f; + CHECK(nir_mse_loss(&pred, &pred, &loss, &grad)); + CHECK_NEAR(loss, 0.0, 1e-7); + double zero[] = {0.0, 0.0, 0.0}; + check_grad(&grad, zero, 3); + nir_tensor_free(&pred); + nir_tensor_free(&grad); +} + +static void test_mse_aliasing(void) { + float pv[] = {1.0f, 2.0f, 3.0f}; + float tv[] = {1.0f, 0.0f, 3.0f}; + nir_tensor_t pred = make_vector(pv, 3); + nir_tensor_t target = make_vector(tv, 3); + float loss = 0.0f; + CHECK(nir_mse_loss(&pred, &target, &loss, &pred)); + double want[] = {0.0, 1.3333333333333333, 0.0}; + check_grad(&pred, want, 3); + nir_tensor_free(&pred); + nir_tensor_free(&target); +} + +static void test_mse_overlap(void) { + float tv[] = {1.0f, 0.0f, 3.0f}; + nir_tensor_t target = make_vector(tv, 3); + float *block = (float *)calloc(4, sizeof(float)); + CHECK(block != NULL); + nir_tensor_t p; + nir_tensor_t g; + memset(&p, 0, sizeof(p)); + memset(&g, 0, sizeof(g)); + size_t shape[1] = {3}; + CHECK(nir_tensor_bind(&p, block, 4, shape, 1)); + CHECK(nir_tensor_bind(&g, block + 1, 3, shape, 1)); + float loss = 0.0f; + CHECK(!nir_mse_loss(&p, &target, &loss, &g)); + free(block); + nir_tensor_free(&target); +} + +static void test_mse_nulls(void) { + float pv[] = {1.0f, 2.0f}; + nir_tensor_t pred = make_vector(pv, 2); + nir_tensor_t target = make_vector(pv, 2); + size_t shape[1] = {2}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL); + float loss = 0.0f; + CHECK(!nir_mse_loss(NULL, &target, &loss, &grad)); + CHECK(!nir_mse_loss(&pred, NULL, &loss, &grad)); + CHECK(!nir_mse_loss(&pred, &target, NULL, &grad)); + CHECK(!nir_mse_loss(&pred, &target, &loss, NULL)); + nir_tensor_free(&pred); + nir_tensor_free(&target); + nir_tensor_free(&grad); +} + +static void test_mse_bad_data(void) { + float pv[] = {1.0f, 2.0f}; + float tv[] = {1.0f, 2.0f, 3.0f}; + float bad[] = {1.0f, (float)NAN}; + nir_tensor_t pred = make_vector(pv, 2); + nir_tensor_t target = make_vector(tv, 3); + nir_tensor_t nonfinite = make_vector(bad, 2); + size_t shape[1] = {2}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL); + float loss = 0.0f; + CHECK(!nir_mse_loss(&pred, &target, &loss, &grad)); + CHECK(!nir_mse_loss(&nonfinite, &pred, &loss, &grad)); + CHECK(!nir_mse_loss(&pred, &nonfinite, &loss, &grad)); + nir_tensor_free(&pred); + nir_tensor_free(&target); + nir_tensor_free(&nonfinite); + nir_tensor_free(&grad); +} + +static void test_spike_count(void) { + float cv[] = {2.0f, 5.0f, 1.0f}; + nir_tensor_t counts = make_vector(cv, 3); + size_t shape[1] = {3}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL); + float loss = 0.0f; + CHECK(nir_spike_count_loss(&counts, 1, &loss, &grad)); + CHECK_NEAR(loss, 0.06588390375742913, 1e-6); + double want[] = {0.0466126225779739, -0.06376044812349424, + 0.01714782554552039}; + check_grad(&grad, want, 3); + CHECK(!nir_spike_count_loss(&counts, 3, &loss, &grad)); + CHECK(!nir_spike_count_loss(NULL, 1, &loss, &grad)); + nir_tensor_free(&counts); + nir_tensor_free(&grad); +} + +static void test_spike_count_matches_ce(void) { + float cv[] = {2.0f, 5.0f, 1.0f}; + nir_tensor_t counts = make_vector(cv, 3); + size_t shape[1] = {3}; + nir_tensor_t grad = nir_tensor_alloc(shape, 1); + nir_tensor_t ce_grad = nir_tensor_alloc(shape, 1); + CHECK(grad.data != NULL && ce_grad.data != NULL); + float loss = 0.0f; + float ce_loss = 0.0f; + CHECK(nir_spike_count_loss(&counts, 1, &loss, &grad)); + CHECK(nir_cross_entropy(&counts, 1, &ce_loss, &ce_grad)); + CHECK(ce_loss == loss); + CHECK(memcmp(grad.data, ce_grad.data, 3 * sizeof(float)) == 0); + nir_tensor_free(&counts); + nir_tensor_free(&grad); + nir_tensor_free(&ce_grad); +} + +int main(void) { + test_mse(); + test_mse_zero(); + test_mse_aliasing(); + test_mse_overlap(); + test_mse_nulls(); + test_mse_bad_data(); + test_spike_count(); + test_spike_count_matches_ce(); + return 0; +} diff --git a/tests/test_schedule.c b/tests/test_schedule.c new file mode 100644 index 0000000..173b671 --- /dev/null +++ b/tests/test_schedule.c @@ -0,0 +1,128 @@ +/* LR schedules: closed-form step/cosine/exponential values, + * boundary steps, optimizer composition, error cases (#46). */ +#include +#include + +#include "nir_c_runtime/schedule.h" +#include "nir_c_runtime/train.h" +#include "test_util.h" + +/* Each case sequences the call before the read: passing `lr` + * in the same expression that writes it is unspecified. */ +static void test_step(void) { + float lr = 0.0f; + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 0, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 9, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 10, &lr)); + CHECK_NEAR(lr, 0.05, 1e-6); + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 19, &lr)); + CHECK_NEAR(lr, 0.05, 1e-6); + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 20, &lr)); + CHECK_NEAR(lr, 0.025, 1e-6); + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 25, &lr)); + CHECK_NEAR(lr, 0.025, 1e-6); +} + +static void test_step_flat(void) { + float lr = 0.0f; + CHECK(nir_lr_step_decay(0.1f, 1.0f, 3, 100, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_step_decay(0.0f, 0.5f, 10, 25, &lr)); + CHECK_NEAR(lr, 0.0, 1e-7); +} + +static void test_cosine(void) { + float lr = 0.0f; + CHECK(nir_lr_cosine(0.1f, 0.0f, 100, 0, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_cosine(0.1f, 0.0f, 100, 50, &lr)); + CHECK_NEAR(lr, 0.05, 1e-6); + CHECK(nir_lr_cosine(0.1f, 0.0f, 100, 100, &lr)); + CHECK_NEAR(lr, 0.0, 1e-6); + /* Torch-exact past t_max: the curve rises again. */ + CHECK(nir_lr_cosine(0.1f, 0.0f, 100, 200, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); +} + +static void test_cosine_floor(void) { + float lr = 0.0f; + CHECK(nir_lr_cosine(0.1f, 0.01f, 100, 0, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_cosine(0.1f, 0.01f, 100, 50, &lr)); + CHECK_NEAR(lr, 0.055, 1e-6); + CHECK(nir_lr_cosine(0.1f, 0.01f, 100, 100, &lr)); + CHECK_NEAR(lr, 0.01, 1e-6); +} + +static void test_exponential(void) { + float lr = 0.0f; + CHECK(nir_lr_exponential(0.1f, 0.5f, 0, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); + CHECK(nir_lr_exponential(0.1f, 0.5f, 1, &lr)); + CHECK_NEAR(lr, 0.05, 1e-6); + CHECK(nir_lr_exponential(0.1f, 0.5f, 3, &lr)); + CHECK_NEAR(lr, 0.0125, 1e-6); + CHECK(nir_lr_exponential(0.1f, 1.0f, 7, &lr)); + CHECK_NEAR(lr, 0.1, 1e-6); +} + +static void test_errors(void) { + float lr = 0.0f; + CHECK(!nir_lr_step_decay(0.1f, 0.5f, 10, 0, NULL)); + CHECK(!nir_lr_step_decay(-0.1f, 0.5f, 10, 0, &lr)); + CHECK(!nir_lr_step_decay(0.1f, -0.5f, 10, 0, &lr)); + CHECK(!nir_lr_step_decay(0.1f, 0.5f, 0, 0, &lr)); + CHECK(!nir_lr_cosine(0.1f, 0.0f, 0, 0, &lr)); + CHECK(!nir_lr_cosine(0.1f, -0.01f, 10, 0, &lr)); + CHECK(!nir_lr_cosine((float)NAN, 0.0f, 10, 0, &lr)); + CHECK(!nir_lr_exponential(0.1f, -0.5f, 0, &lr)); + CHECK(!nir_lr_exponential((float)INFINITY, 0.5f, 0, &lr)); +} + +/* Schedules drive both optimizers through their `lr` fields, + * keyed by the checkpoint step counter (0 for a fresh run). */ +static float resumed_lr(void) { + float lr = 0.0f; + CHECK(nir_lr_step_decay(0.1f, 0.5f, 10, 10, &lr)); + return lr; +} + +static void test_sgd_compose(void) { + char *error = NULL; + nir_sgd_t sgd; + CHECK(nir_sgd_init(&sgd, 1, 0.1f, 0.0f, 0.0f, &error)); + sgd.lr = resumed_lr(); + float w[] = {1.0f}; + float g[] = {2.0f}; + CHECK(nir_sgd_step(&sgd, w, g, 1)); + CHECK_NEAR(w[0], 0.9, 1e-6); + nir_sgd_free(&sgd); +} + +static void test_adam_compose(void) { + char *error = NULL; + nir_adam_t adam; + CHECK(nir_adam_init( + &adam, 1, 0.1f, 0.9f, 0.999f, 1e-8f, 0.0f, &error + )); + adam.lr = resumed_lr(); + float w[] = {1.0f}; + float g[] = {0.1f}; + CHECK(nir_adam_step(&adam, w, g, 1)); + CHECK_NEAR(w[0], 0.95, 1e-6); + nir_adam_free(&adam); +} + +int main(void) { + test_step(); + test_step_flat(); + test_cosine(); + test_cosine_floor(); + test_exponential(); + test_errors(); + test_sgd_compose(); + test_adam_compose(); + return 0; +}