diff --git a/CMakeLists.txt b/CMakeLists.txt index 7735dc2..884b23f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,15 @@ foreach(test_name IN LISTS TENSORLIB_TESTS) endforeach() # Each policy case starts a fresh process with an isolated ENV snapshot. +if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + add_executable(test_autograd_allocation_failure + tests/unit/autograd/test_autograd_allocation_failure.c) + target_link_libraries(test_autograd_allocation_failure PRIVATE tensorlib) + target_link_options(test_autograd_allocation_failure PRIVATE + "-Wl,--wrap=malloc,--wrap=calloc,--wrap=realloc") + add_test(NAME test_autograd_allocation_failure COMMAND test_autograd_allocation_failure) +endif() + add_executable(test_parallel tests/unit/tensor/test_parallel.c) target_include_directories(test_parallel PRIVATE tests/fixtures) target_link_libraries(test_parallel PRIVATE tensorlib) diff --git a/benchmarks/bench_backward.c b/benchmarks/bench_backward.c index 4b869c3..1f0beaa 100644 --- a/benchmarks/bench_backward.c +++ b/benchmarks/bench_backward.c @@ -11,7 +11,9 @@ typedef enum { LINEAR_DWEIGHT, LINEAR_DWEIGHT_DIRECT, LINEAR_DBIAS, - LINEAR_PACK + LINEAR_PACK, + LINEAR_FORWARD, + LINEAR_FORWARD_REPACK } linear_gradient_kind; typedef struct { @@ -73,6 +75,9 @@ static int init_linear_context(linear_gradient_context* context, if (kind == LINEAR_DINPUT) { context->packed_weight = t_pack_matmul_rhs(context->weight); if (context->packed_weight == NULL) return 1; + } else if (kind == LINEAR_FORWARD) { + context->packed_weight = t_pack_matmul_rhs_transposed(context->weight); + if (context->packed_weight == NULL) return 1; } return 0; } @@ -83,7 +88,14 @@ static int linear_gradient_operation(void* opaque, double* checksum) tensor* first = NULL; tensor* second = NULL; tensor* output = NULL; - if (context->kind == LINEAR_DINPUT) { + if (context->kind == LINEAR_FORWARD || context->kind == LINEAR_FORWARD_REPACK) { + if (context->kind == LINEAR_FORWARD_REPACK) { + tensor_mark_modified(context->weight); + t_free_matmul_packed_rhs(context->packed_weight); + context->packed_weight = t_pack_matmul_rhs_transposed(context->weight); + } + output = t_matmul_packed_rhs(context->input, context->packed_weight); + } else if (context->kind == LINEAR_DINPUT) { output = t_matmul_packed_rhs(context->output_gradient, context->packed_weight); } else if (context->kind == LINEAR_DWEIGHT) { @@ -240,11 +252,21 @@ int bench_run_backward_matrix_suite(const bench_options* options, FILE* csv) {"linear_attention_output", 192, 192}, {"linear_mlp_expand", 192, 768}, {"linear_mlp_project", 768, 192}, - {"linear_vocabulary", 192, 256} + {"linear_vocabulary", 192, 256}, + {"linear_tail", 193, 257}, + {"linear_narrow", 192, 17}, + {"linear_skinny", 192, 7} }; int status = 0; printf("Backward matrix components\n"); for (size_t index = 0; index < sizeof(shapes) / sizeof(shapes[0]); ++index) { + for (int thread_index = 0; thread_index < options->thread_count; ++thread_index) { + int threads = options->threads[thread_index]; + status |= run_linear_gradient(options, csv, &shapes[index], + LINEAR_FORWARD, "forward", "packed-weight", threads); + status |= run_linear_gradient(options, csv, &shapes[index], + LINEAR_FORWARD_REPACK, "forward_repack", "pack+multiply", threads); + } status |= run_linear_gradient(options, csv, &shapes[index], LINEAR_DINPUT, "dinput", "packed-weight", options->threads[0]); diff --git a/benchmarks/bench_harness.c b/benchmarks/bench_harness.c index ed240ac..ca3a7cc 100644 --- a/benchmarks/bench_harness.c +++ b/benchmarks/bench_harness.c @@ -33,6 +33,11 @@ bench_profile bench_profile_named(const char* name) profile.sample_count = 15; profile.minimum_sample_seconds = 0.1; profile.warmup_seconds = 0.25; + } else if (name != NULL && strcmp(name, "fixed") == 0) { + profile.profile = "fixed"; + profile.sample_count = 5; + profile.minimum_sample_seconds = 0.0; + profile.warmup_seconds = 0.0; } else if (name != NULL && strcmp(name, "smoke") == 0) { profile.profile = "smoke"; profile.sample_count = 1; @@ -88,7 +93,9 @@ int bench_measure_with_reset(bench_operation operation, } warmup_elapsed += elapsed; } - if (run_iterations(operation, context, 1, &checksum, &elapsed) != 0) { + /* Fixed training sequences must not advance model/RNG state in a probe. */ + if (strcmp(profile->profile, "fixed") != 0 && + run_iterations(operation, context, 1, &checksum, &elapsed) != 0) { return 1; } if (elapsed > 0.0 && elapsed < profile->minimum_sample_seconds) { diff --git a/benchmarks/bench_nn.c b/benchmarks/bench_nn.c index c3fd7c9..6efb362 100644 --- a/benchmarks/bench_nn.c +++ b/benchmarks/bench_nn.c @@ -19,10 +19,12 @@ typedef struct { nn_adamw* adamw; nn_rng rng; int train; + int diagnostics; int component_backward; double* phase_samples[6]; int phase_sample_count; int phase_sample_capacity; + unsigned long measured_calls; tensor_alloc_stats allocation_stats; ag_backward_stats backward_stats; size_t allocation_baseline_bytes; @@ -63,6 +65,11 @@ static tensor* make_targets(int count, int classes) return targets; } +static double phase_now(const nn_bench_context* context) +{ + return context->diagnostics ? bench_now_seconds() : 0.0; +} + static int nn_operation(void* opaque, double* checksum) { nn_bench_context* context = (nn_bench_context*)opaque; @@ -73,33 +80,33 @@ static int nn_operation(void* opaque, double* checksum) double phase_times[PHASE_COUNT] = {0}; if (context->train) { - started = bench_now_seconds(); + started = phase_now(context); if (context->sgd != NULL) { nn_sgd_zero_grad(context->sgd); } else if (context->adamw != NULL) { nn_adamw_zero_grad(context->adamw); } - phase_times[PHASE_ZERO_GRAD] = bench_now_seconds() - started; + phase_times[PHASE_ZERO_GRAD] = phase_now(context) - started; } if (context->component_backward) nn_module_zero_grad(context->module); - started = bench_now_seconds(); + started = phase_now(context); output = context->forward(context->model, context->input); if (context->train) { - phase_times[PHASE_FORWARD] = bench_now_seconds() - started; + phase_times[PHASE_FORWARD] = phase_now(context) - started; } if (output == NULL) goto cleanup; if (context->train) { ag_backward_options backward_options = ag_backward_default_options(); backward_options.retention = AG_GRAD_RETAIN_LEAVES; - started = bench_now_seconds(); + started = phase_now(context); loss = nn_cross_entropy(output, context->targets); - phase_times[PHASE_LOSS] = bench_now_seconds() - started; + phase_times[PHASE_LOSS] = phase_now(context) - started; if (loss == NULL) goto cleanup; - started = bench_now_seconds(); + started = phase_now(context); if (ag_backward_ex(loss, &backward_options) != 0) goto cleanup; - phase_times[PHASE_BACKWARD] = bench_now_seconds() - started; - started = bench_now_seconds(); + phase_times[PHASE_BACKWARD] = phase_now(context) - started; + started = phase_now(context); if (context->sgd != NULL) { if (nn_sgd_step(context->sgd) != 0) goto cleanup; } else { @@ -107,7 +114,7 @@ static int nn_operation(void* opaque, double* checksum) goto cleanup; } } - phase_times[PHASE_ADAMW] = bench_now_seconds() - started; + phase_times[PHASE_ADAMW] = phase_now(context) - started; *checksum += loss->value->storage->data[loss->value->offset]; } else if (context->component_backward) { tensor* seed = t_alloc(output->value->ndim, output->value->dims); @@ -125,14 +132,15 @@ static int nn_operation(void* opaque, double* checksum) *checksum += output->value->storage->data[output->value->offset]; } status = 0; + if (context->train) ++context->measured_calls; cleanup: - if (context->train) started = bench_now_seconds(); + if (context->train) started = phase_now(context); ag_tensor_release(loss); ag_tensor_release(output); if (context->component_backward) nn_module_zero_grad(context->module); if (context->train) { - phase_times[PHASE_GRAPH_RELEASE] = bench_now_seconds() - started; + phase_times[PHASE_GRAPH_RELEASE] = phase_now(context) - started; if (context->phase_sample_count < context->phase_sample_capacity) { for (int phase = 0; phase < PHASE_COUNT; ++phase) { context->phase_samples[phase][context->phase_sample_count] = @@ -148,6 +156,7 @@ static void reset_phase_samples(void* opaque) { nn_bench_context* context = (nn_bench_context*)opaque; context->phase_sample_count = 0; + context->measured_calls = 0; tensor_alloc_stats_reset_counters(); } @@ -158,9 +167,27 @@ static void report_allocation_stats(const bench_options* options, const nn_bench_context* context) { char name[64]; - double calls = context->phase_sample_count > 0 - ? (double)context->phase_sample_count : 1.0; + double calls = context->measured_calls > 0 + ? (double)context->measured_calls : 1.0; const tensor_alloc_stats* stats = &context->allocation_stats; + static const char* kinds[] = {"metadata", "graph", "matmul_buffer"}; + for (int kind = 0; kind < TENSOR_ALLOC_AUX_KINDS; ++kind) { + snprintf(name, sizeof(name), "%s_%s_allocations", + context->metric_prefix, kinds[kind]); + bench_record_scalar(options, csv, "nn_phase", name, + context->metric_shape, "instrumented-auxiliary", "alloc/call", + requested_threads, measured_threads, + (double)stats->auxiliary_allocations[kind] / calls); + snprintf(name, sizeof(name), "%s_%s_bytes", context->metric_prefix, kinds[kind]); + bench_record_scalar(options, csv, "nn_phase", name, + context->metric_shape, "instrumented-auxiliary", "bytes/call", + requested_threads, measured_threads, + (double)stats->auxiliary_bytes[kind] / calls); + } + snprintf(name, sizeof(name), "%s_copied_bytes", context->metric_prefix); + bench_record_scalar(options, csv, "nn_phase", name, + context->metric_shape, "clone+projection-copy", "bytes/call", + requested_threads, measured_threads, (double)stats->copied_bytes / calls); snprintf(name, sizeof(name), "%s_allocations", context->metric_prefix); bench_record_scalar(options, csv, "nn_phase", name, context->metric_shape, "isolated-phase", @@ -214,6 +241,16 @@ static void report_backward_stats(const bench_options* options, requested_threads, measured_threads, stats->operation_seconds[operation] * 1000.0 / (double)stats->operation_calls[operation]); + snprintf(name, sizeof(name), "backward_op_%s_copied_bytes", names[operation]); + bench_record_scalar(options, csv, "nn_backward", name, + context->metric_shape, "clone+projection-copy", "bytes/backward", + requested_threads, measured_threads, + (double)stats->operation_copied_bytes[operation]); + snprintf(name, sizeof(name), "backward_op_%s_allocated_bytes", names[operation]); + bench_record_scalar(options, csv, "nn_backward", name, + context->metric_shape, "storage+instrumented-auxiliary", "bytes/backward", + requested_threads, measured_threads, + (double)stats->operation_allocated_bytes[operation]); } static const char* engine_names[] = { "backward_graph_traversal", "backward_shape_reduction", @@ -322,6 +359,7 @@ static int run_nn_case(const bench_options* options, suite, name, shape, layout, metric, items, 1, nn_operation, context }; + context->diagnostics = options->diagnostics; benchmark.reset = context->train ? reset_phase_samples : NULL; return bench_execute_case(options, csv, &benchmark, threads, result); } @@ -504,7 +542,7 @@ static int run_mlp(const bench_options* options, FILE* csv, int batch, int train train ? "mnist_mlp_train_step" : "mnist_mlp_forward", "[Bx784]->[Bx10]", train ? "forward+loss+backward+sgd" : "forward;graph-build", "samples/s", (double)batch, - 1, &context, &result); + options->threads[0], &context, &result); destroy_context(&context); return status == 1; } @@ -573,20 +611,22 @@ static int run_decoder_case(const bench_options* options, nn_bench_context context; bench_measurement local_result; bench_measurement* result = measurement == NULL ? &local_result : measurement; - if (train) { + int diagnostics = train && options->diagnostics; + if (diagnostics) { tensor_alloc_stats_enable(1); tensor_alloc_stats_reset(); } if (setup_decoder(&context, batch, time, channels, layers, train) != 0) { - if (train) tensor_alloc_stats_enable(0); + if (diagnostics) tensor_alloc_stats_enable(0); destroy_context(&context); return 1; } - if (train && allocate_phase_samples(&context, options->profile.sample_count) != 0) { + if (diagnostics && allocate_phase_samples(&context, options->profile.sample_count) != 0) { destroy_context(&context); + tensor_alloc_stats_enable(0); return 1; } - if (train) { + if (diagnostics) { tensor_alloc_stats_read(&context.allocation_stats); context.allocation_baseline_bytes = context.allocation_stats.live_bytes; @@ -611,7 +651,7 @@ static int run_decoder_case(const bench_options* options, case_name, shape, train ? "forward+loss+backward+adamw" : "forward;graph-build", "tokens/s", (double)(batch * time), threads, &context, result); - if (status == 0 && train && + if (status == 0 && diagnostics && (strcmp(suite, "nn") == 0 || command_model)) { int measured_threads = bench_configure_threads(threads); tensor_alloc_stats_read(&context.allocation_stats); @@ -630,7 +670,7 @@ static int run_decoder_case(const bench_options* options, } } destroy_context(&context); - if (train) { + if (diagnostics) { tensor_alloc_stats_read(&context.allocation_stats); if (context.allocation_stats.live_bytes != 0) status = 1; tensor_alloc_stats_enable(0); @@ -741,3 +781,20 @@ int bench_run_command_suite(const bench_options* options, FILE* csv) printf("\n"); return status; } + +int bench_run_training_suite(const bench_options* options, FILE* csv) +{ + int smoke = strcmp(options->profile.profile, "smoke") == 0; + int status = 0; + for (int index = 0; index < options->thread_count; ++index) { + bench_options single = *options; + single.threads[0] = options->threads[index]; + single.thread_count = 1; + status |= run_mlp(&single, csv, smoke ? 2 : 64, 1); + status |= run_decoder_case(&single, csv, "nn", smoke ? 1 : 4, + smoke ? 8 : 128, smoke ? 24 : 192, smoke ? 1 : 4, + 1, single.threads[0], NULL) == 1; + status |= bench_run_command_suite(&single, csv); + } + return status; +} diff --git a/benchmarks/bench_runner.c b/benchmarks/bench_runner.c index 070d5c2..dc7e526 100644 --- a/benchmarks/bench_runner.c +++ b/benchmarks/bench_runner.c @@ -136,6 +136,7 @@ int bench_run_command_suite(const bench_options* options, FILE* csv); int bench_run_backward_matrix_suite(const bench_options* options, FILE* csv); int bench_run_scaling_suite(const bench_options* options, FILE* csv); int bench_run_policy_suite(const bench_options* options, FILE* csv); +int bench_run_training_suite(const bench_options* options, FILE* csv); int bench_run_suites(const bench_options* options, FILE* csv) { @@ -152,6 +153,9 @@ int bench_run_suites(const bench_options* options, FILE* csv) status |= bench_run_nn_suite(options, csv); status |= bench_run_backward_matrix_suite(options, csv); } + if (strcmp(options->suite, "matmul") == 0) { + status |= bench_run_backward_matrix_suite(options, csv); + } if (suite_selected(options->suite, "command")) { status |= bench_run_command_suite(options, csv); } @@ -161,5 +165,8 @@ int bench_run_suites(const bench_options* options, FILE* csv) if (suite_selected(options->suite, "policy")) { status |= bench_run_policy_suite(options, csv); } + if (strcmp(options->suite, "training") == 0) { + status |= bench_run_training_suite(options, csv); + } return status; } diff --git a/benchmarks/bench_runner.h b/benchmarks/bench_runner.h index 6255877..2a23b76 100644 --- a/benchmarks/bench_runner.h +++ b/benchmarks/bench_runner.h @@ -12,6 +12,7 @@ typedef struct { const char* suite; int threads[BENCH_MAX_THREADS]; int thread_count; + int diagnostics; } bench_options; typedef struct { diff --git a/benchmarks/bench_tensorlib.c b/benchmarks/bench_tensorlib.c index 409ea5b..c1ed7f9 100644 --- a/benchmarks/bench_tensorlib.c +++ b/benchmarks/bench_tensorlib.c @@ -12,9 +12,9 @@ static void usage(const char* program) { - printf("Usage: %s [--profile quick|full] [--suite NAME] " - "[--threads LIST] [--csv PATH] [--smoke]\n", program); - printf("Suites: kernels, autograd, nn, command, scaling, policy, all\n"); + printf("Usage: %s [--profile quick|full|fixed] [--suite NAME] " + "[--threads LIST] [--csv PATH] [--no-diagnostics] [--smoke]\n", program); + printf("Suites: kernels, autograd, nn, command, training, matmul, scaling, policy, all\n"); } static int available_threads(void) @@ -64,7 +64,8 @@ static int valid_suite(const char* suite) return strcmp(suite, "all") == 0 || strcmp(suite, "kernels") == 0 || strcmp(suite, "autograd") == 0 || strcmp(suite, "nn") == 0 || strcmp(suite, "command") == 0 || strcmp(suite, "scaling") == 0 || - strcmp(suite, "policy") == 0; + strcmp(suite, "policy") == 0 || strcmp(suite, "training") == 0 || + strcmp(suite, "matmul") == 0; } static const char* compiler_name(void) @@ -108,16 +109,20 @@ int main(int argc, char** argv) options.profile = bench_profile_named("quick"); options.suite = "all"; + options.diagnostics = 1; default_threads(&options); for (int index = 1; index < argc; ++index) { if (strcmp(argv[index], "--help") == 0) { usage(argv[0]); return EXIT_SUCCESS; + } else if (strcmp(argv[index], "--no-diagnostics") == 0) { + options.diagnostics = 0; } else if (strcmp(argv[index], "--smoke") == 0) { options.profile = bench_profile_named("smoke"); } else if (strcmp(argv[index], "--profile") == 0 && index + 1 < argc) { const char* name = argv[++index]; - if (strcmp(name, "quick") != 0 && strcmp(name, "full") != 0) { + if (strcmp(name, "quick") != 0 && strcmp(name, "full") != 0 && + strcmp(name, "fixed") != 0) { usage(argv[0]); return EXIT_FAILURE; } diff --git a/docs/benchmarks.md b/docs/benchmarks.md index c017954..800fceb 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -58,12 +58,31 @@ Run `build-bench/bench_tensorlib --help` for the accepted command-line options. | `smoke` | 1 | none | none | Correctness and portability checks only | | `quick` | 5 | 20 ms | 20 ms | Local development and regression triage | | `full` | 15 | 100 ms | 250 ms | Results intended for comparison or publication | +| `fixed` | 5 | none | none | Identical five-step training sequences across implementations | -The harness first times one operation, then batches up to 1,000,000 iterations +The quick/full harness first times one operation, then batches up to 1,000,000 iterations per sample to reach the profile's minimum duration. It reports time per operation. The median is the primary result; p95 is useful for spotting noisy runs. Do not treat `smoke` timings as performance measurements. +The `fixed` profile executes exactly five measured calls, with no calibration +call or warm-up. Use it for matched training comparisons so both processes +start from the same seeded state and perform the same optimizer/RNG sequence. +Repeat in fresh processes; the five calls include the first-step packing cost. + +For primary training timings use `--suite training --profile fixed +--no-diagnostics`. This measures MNIST MLP (batch 64), TinyLM (batch 4), and +CommandLM (batches 1 and 16) at each requested thread count. It disables phase +clocks, allocation counters, and the extra backward profiling step. Run again +without `--no-diagnostics` for a separate diagnostic profile; do not compare +its throughput directly with the uninstrumented measurements. + +The Python runner records the executable SHA-256, available CMake settings, +CPU topology, inherited process affinity, thread environment, and checkout +revision/status. Supply `--source-revision REV` from the clean source used for +the build; without it the binary's source revision is explicitly `unverified`. +The current checkout revision alone cannot identify an older executable. + Run the full profile with: ```sh @@ -73,13 +92,16 @@ make benchmark-full BUILD_DIR=build-bench \ ## Suites and measurement boundaries -Select one suite with `--suite kernels|autograd|nn|scaling`, or use `all`. +Select one suite with `--suite kernels|autograd|nn|command|training|matmul|scaling|policy`, +or use `all`. Use `training` for the designated model acceptance workloads. | Suite | Representative cases | What the timed region includes | |-------|----------------------|--------------------------------| | `kernels` | allocation, views, copies, broadcasting, GELU, reductions, gather, matmul, RHS packing | Public API call and output allocation/free | | `autograd` | composed elementwise forward; forward plus backward | Dynamic graph construction; backward case also includes gradient computation, zeroing, and graph cleanup | | `nn` | Linear, LayerNorm, attention, decoder block, MNIST MLP, TinyLM | Eager forward graph construction; training rows include loss, backward, optimizer step, and zero-grad | +| `training` | MNIST MLP batch 64, TinyLM batch 4, CommandLM batches 1/16 | Complete training calls at every requested thread count, including release and weight updates | +| `matmul` | Linear forward, repacking, dinput/dweight, packing, narrow/tail shapes and attention gradients | Multiplication or the explicitly named packing/copy/reduction boundary; reusable inputs outside timing | | `scaling` | large add, GELU, square matmul, TinyLM forward/train | The same workload across the requested OpenMP thread ladder, plus derived speedup data for tensor kernels | The Python runner records `TENSORLIB_*` and `OMP_*` values in the CSV @@ -91,9 +113,16 @@ remain inside it. Forward-only NN rows are labeled `forward;graph-build`: TensorLib has no no-grad execution mode, and these rows should not be described as inference-only latency. -The `kernels`, `autograd`, and `nn` suites currently run their cases with one -requested thread. Use the `scaling` suite to study thread count. Reductions and -gather are serial today and identify that fact in the `layout` column. +Most `kernels`, `autograd`, and component `nn` cases use the first requested +thread count. Use `training` or `scaling` for complete model ladders. In `matmul`, +forward, forward-repack and direct dweight span the ladder; dinput, legacy +dweight, packing, bias reduction and attention use its first count. Reductions +and gather are serial today and identify that fact in the `layout` column. + +`forward_repack` invalidates and rebuilds a weight pack every call; it does not +perform an optimizer update. The `training` suite supplies that acceptance +boundary. The isolated projection shapes use 4×128 input rows, including width +193/output 257 tails and outputs 17/7; they are not all CommandLM shapes. ## Thread scaling @@ -155,7 +184,7 @@ The native and reference runners share these columns: |--------|---------| | `suite`, `case` | Workload group and stable case name | | `shape`, `layout` | Symbolic dimensions and relevant memory/execution layout | -| `profile` | `smoke`, `quick`, or `full` | +| `profile` | `smoke`, `quick`, `full`, or `fixed` | | `requested_threads`, `actual_threads` | Requested OpenMP/library setting and observed team size | | `median_seconds`, `p95_seconds` | Per-operation timing statistics | | `metric`, `value` | Unit and derived throughput, or milliseconds per call | @@ -170,6 +199,16 @@ scaling speedup is currently printed to the console but is not emitted as a separate derived CSV row, so calculate it from its `status=ok` timing rows when processing CSV data. +The fixed training profile also appears as `profile=fixed`. Diagnostic rows +report storage allocation/live/peak bytes, requested auxiliary allocation counts +and bytes for metadata/graphs/matmul scratch, and logical clone/QKV assembly +copy bytes. Auxiliary reallocations count request sizes, not net live bytes. +The live/peak figures cover tensor storage only. These counters exclude some +NN/optimizer allocations, allocator overhead and other memory traffic. +Backward per-operation byte deltas cover the operation callback; engine merges +and reductions have separate timing rows. Run primary throughput separately +with `--no-diagnostics`. + Metric formulas are conventional effective rates: - Matmul uses `2 * M * N * K / median_seconds` FLOP/s, multiplied by batch @@ -223,4 +262,3 @@ and use the shared harness rather than adding a new timer. If a reference case is added, keep its shape and timed allocation boundary aligned and document any semantic difference. Validate harness changes with `test_bench_harness` and run the smoke benchmark before collecting timings. - diff --git a/include/tensorlib/autograd_internal.h b/include/tensorlib/autograd_internal.h index dcd8586..0775a7e 100644 --- a/include/tensorlib/autograd_internal.h +++ b/include/tensorlib/autograd_internal.h @@ -8,6 +8,8 @@ typedef struct { double operation_seconds[AG_BACKWARD_OP_COUNT]; unsigned long operation_calls[AG_BACKWARD_OP_COUNT]; + unsigned long long operation_copied_bytes[AG_BACKWARD_OP_COUNT]; + unsigned long long operation_allocated_bytes[AG_BACKWARD_OP_COUNT]; double traversal_seconds; double reduction_seconds; double accumulation_seconds; diff --git a/scripts/compare_thread_profiles.py b/scripts/compare_thread_profiles.py new file mode 100644 index 0000000..1738214 --- /dev/null +++ b/scripts/compare_thread_profiles.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Compare explicitly defined thread profiles in rotated fresh processes.""" + +import argparse +import csv +import json +import math +import os +from pathlib import Path +import statistics +import subprocess +import sys + +from compare_training import read_rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("executable", "source-revision", "profiles", "output"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + if args.repeats < 3: + parser.error("at least three fresh processes per profile are required") + profiles = json.loads(Path(args.profiles).read_text()) + names = [p["name"] for p in profiles] + if not names or any(not n for n in names) or len(set(names)) != len(names): + parser.error("profile names must be nonempty and unique") + for profile in profiles: + if profile["reference"] not in names or int(profile["threads"]) < 1: + parser.error("each profile requires a valid reference and positive thread count") + if any(not k.startswith(("OMP_", "TENSORLIB_")) for k in profile["environment"]): + parser.error("profile settings must be OMP_* or TENSORLIB_*") + output = Path(args.output) + output.mkdir(parents=True, exist_ok=False) + (output / "profiles.json").write_text(json.dumps(profiles, indent=2) + "\n") + inherited = {k: v for k, v in os.environ.items() if not k.startswith(("OMP_", "TENSORLIB_"))} + results = {} + for repeat in range(args.repeats): + order = profiles[repeat % len(profiles):] + profiles[:repeat % len(profiles)] + if repeat % 2: + order.reverse() + for profile in order: + environment = dict(inherited, OMP_DYNAMIC="FALSE") + environment.update(profile["environment"]) + path = output / f"profile{names.index(profile['name'])}-run{repeat}.csv" + command = [sys.executable, str(Path(__file__).with_name("run_benchmarks.py")), + "--executable", args.executable, "--source-revision", args.source_revision, + "--suite", "training", "--profile", "fixed", "--no-diagnostics", + "--threads", str(profile["threads"]), "--csv", str(path)] + print(f"Repeat {repeat + 1}: {profile['name']}", flush=True) + with path.with_suffix(".log").open("w") as log: + subprocess.run(command, env=environment, stdout=log, + stderr=subprocess.STDOUT, check=True) + results[(profile["name"], repeat)] = {k[:5]: r for k, r in read_rows(path).items()} + summaries = [] + canonical = next(iter(results.values())) + for rows in results.values(): + if rows.keys() != canonical.keys(): + raise ValueError("Training inventories differ") + for key, row in rows.items(): + for field in ("cpu", "os", "executable_sha256", "source_revision", "process_affinity"): + if row[field] != canonical[key][field]: + raise ValueError(f"Binary or host changed: {field}") + if row["profile"] != "fixed" or row["iterations"] != canonical[key]["iterations"]: + raise ValueError("Training sequences differ") + if not math.isclose(float(row["checksum"]), float(canonical[key]["checksum"]), + rel_tol=1e-5, abs_tol=1e-5): + raise ValueError("Training loss checksums differ") + for profile in profiles: + for key in canonical: + times, references, ratios = [], [], [] + for repeat in range(args.repeats): + after = float(results[(profile["name"], repeat)][key]["median_seconds"]) + before = float(results[(profile["reference"], repeat)][key]["median_seconds"]) + times.append(after * 1000) + references.append(before * 1000) + ratios.append(after / before) + median = statistics.median(ratios) + summaries.append({"profile": profile["name"], "reference": profile["reference"], + "case": key[1], "threads": profile["threads"], + "median_ms": statistics.median(times), + "reference_ms": statistics.median(references), + "change_percent": (median - 1) * 100, + "ratio_min": min(ratios), "ratio_max": max(ratios), + "spread_percent": (max(times) - min(times)) / + statistics.median(times) * 100, + "regression_over_5_percent": median > 1.05}) + with (output / "summary.csv").open("w", newline="") as destination: + writer = csv.DictWriter(destination, fieldnames=list(summaries[0])) + writer.writeheader() + writer.writerows(summaries) + + +if __name__ == "__main__": + main() diff --git a/scripts/compare_training.py b/scripts/compare_training.py new file mode 100644 index 0000000..d8d6ba0 --- /dev/null +++ b/scripts/compare_training.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Alternate fresh-process training runs and summarize paired latency ratios.""" + +import argparse +import csv +import math +import os +from pathlib import Path +import statistics +import subprocess +import sys + + +def read_rows(path): + with path.open(newline="", encoding="utf-8") as source: + rows = list(csv.DictReader(source)) + result = {} + for row in rows: + if row["status"] != "ok": + raise ValueError(f"Failed/skipped benchmark in {path}: {row['case']}") + key = tuple(row[field] for field in + ("suite", "case", "shape", "layout", "profile", "requested_threads")) + if key in result: + raise ValueError(f"Duplicate benchmark in {path}: {key}") + latency = float(row["median_seconds"]) + if not math.isfinite(latency) or latency <= 0: + raise ValueError(f"Invalid latency in {path}: {key}") + result[key] = row + if not result: + raise ValueError(f"No measurements in {path}") + return result + + +def summarize(pairs): + baseline, candidate = pairs[0] + keys = baseline.keys() + for left, right in pairs: + if left.keys() != keys or right.keys() != keys: + raise ValueError("Benchmark inventories differ") + summaries = [] + for key in keys: + if key[4] != "fixed": + raise ValueError("Paired training comparisons require the fixed profile") + ratios, before, after = [], [], [] + for left, right in pairs: + a, b = left[key], right[key] + for field in ("os", "cpu", "thread_environment", "process_affinity"): + if a.get(field) != b.get(field): + raise ValueError(f"Measurement environments differ: {field}") + for rows, reference in ((left, baseline), (right, candidate)): + for field in ("executable_sha256", "source_revision"): + if rows[key].get(field) != reference[key].get(field): + raise ValueError(f"Executable changed between repeats: {field}") + if a["iterations"] != b["iterations"]: + raise ValueError(f"Training sequences differ: {key}") + if not math.isclose(float(a["checksum"]), float(b["checksum"]), + rel_tol=1e-5, abs_tol=1e-5): + raise ValueError(f"Training loss checksums differ: {key}") + before.append(float(a["median_seconds"])) + after.append(float(b["median_seconds"])) + ratios.append(after[-1] / before[-1]) + median = statistics.median(ratios) + summaries.append({ + "case": key[1], "shape": key[2], "threads": key[5], + "baseline_ms": statistics.median(before) * 1000, + "candidate_ms": statistics.median(after) * 1000, + "paired_ratio": median, "ratio_min": min(ratios), + "ratio_max": max(ratios), "change_percent": (median - 1) * 100, + "baseline_spread_percent": (max(before) - min(before)) / + statistics.median(before) * 100, + "candidate_spread_percent": (max(after) - min(after)) / + statistics.median(after) * 100, + "regression_over_5_percent": median > 1.05, + }) + return summaries + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("baseline", "candidate", "baseline-revision", "candidate-revision", "output"): + parser.add_argument("--" + name, required=True) + parser.add_argument("--threads", default="1,6,16") + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--env", action="append", default=[], metavar="NAME=VALUE") + args = parser.parse_args() + if args.repeats < 3: + parser.error("at least three fresh-process pairs are required") + output = Path(args.output) + output.mkdir(parents=True, exist_ok=False) + environment = {key: value for key, value in os.environ.items() + if not key.startswith(("OMP_", "TENSORLIB_"))} + environment["OMP_DYNAMIC"] = "FALSE" + for setting in args.env: + name, separator, value = setting.partition("=") + if not separator or not name.startswith(("OMP_", "TENSORLIB_")): + parser.error("--env requires an OMP_* or TENSORLIB_* setting") + environment[name] = value + runner = Path(__file__).with_name("run_benchmarks.py") + pairs = [] + for repeat in range(args.repeats): + paths = {} + order = ("baseline", "candidate") if repeat % 2 == 0 else ("candidate", "baseline") + for variant in order: + path = output / f"{variant}-{repeat}.csv" + command = [sys.executable, str(runner), "--executable", getattr(args, variant), + "--source-revision", getattr(args, variant + "_revision"), + "--suite", "training", "--profile", "fixed", "--no-diagnostics", + "--threads", args.threads, "--csv", str(path)] + print(f"Pair {repeat + 1}: {variant}", flush=True) + with path.with_suffix(".log").open("w", encoding="utf-8") as log: + subprocess.run(command, env=environment, stdout=log, + stderr=subprocess.STDOUT, check=True) + paths[variant] = path + pairs.append((read_rows(paths["baseline"]), read_rows(paths["candidate"]))) + rows = summarize(pairs) + with (output / "comparison.csv").open("w", newline="", encoding="utf-8") as destination: + writer = csv.DictWriter(destination, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + for row in rows: + print(f"{row['case']} threads={row['threads']}: {row['change_percent']:+.2f}% " + f"(paired ratios {row['ratio_min']:.3f}..{row['ratio_max']:.3f})") + return 1 if any(row["regression_over_5_percent"] for row in rows) else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_benchmarks.py b/scripts/run_benchmarks.py index 2a53b60..6c1d15c 100755 --- a/scripts/run_benchmarks.py +++ b/scripts/run_benchmarks.py @@ -3,6 +3,7 @@ import argparse import csv +import hashlib import json import os import platform @@ -10,6 +11,36 @@ import subprocess import sys import tempfile +from pathlib import Path + + +def command_output(command): + try: + result = subprocess.run(command, capture_output=True, text=True, check=False) + return result.stdout.strip() if result.returncode == 0 else "unavailable" + except OSError: + return "unavailable" + + +def build_metadata(executable, revision): + path = Path(executable).resolve() + cache = path.parent / "CMakeCache.txt" + settings = {} + if cache.exists(): + for line in cache.read_text(encoding="utf-8").splitlines(): + if re.match(r"(?:CMAKE_(?:BUILD_TYPE|C_COMPILER|C_FLAGS.*)|TENSORLIB_.*):", line): + key, value = line.split("=", 1) + settings[key] = value + return { + "source_revision": revision or "unverified", + "checkout_revision": command_output(["git", "rev-parse", "HEAD"]), + "checkout_status": command_output(["git", "status", "--porcelain", "--untracked-files=no"]), + "executable_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "build_settings": json.dumps(settings, sort_keys=True), + "cpu_topology": command_output(["lscpu", "-e=CPU,CORE,SOCKET,NODE,ONLINE"]), + "process_affinity": str(sorted(os.sched_getaffinity(0))) + if hasattr(os, "sched_getaffinity") else "unavailable", + } def cpu_model(): @@ -33,12 +64,14 @@ def cpu_model(): def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--executable", default="build/bench_tensorlib") - parser.add_argument("--profile", choices=("quick", "full"), default="quick") - parser.add_argument("--suite", choices=("all", "kernels", "autograd", "nn", "command", "scaling", "policy"), + parser.add_argument("--profile", choices=("quick", "full", "fixed"), default="quick") + parser.add_argument("--suite", choices=("all", "kernels", "autograd", "nn", "command", "training", "matmul", "scaling", "policy"), default="all") parser.add_argument("--threads", help="comma-separated OpenMP thread ladder") parser.add_argument("--csv", default="benchmark-results.csv") parser.add_argument("--smoke", action="store_true") + parser.add_argument("--no-diagnostics", action="store_true") + parser.add_argument("--source-revision", help="verified revision used to build the executable") return parser.parse_args() @@ -53,6 +86,7 @@ def main(): if key.startswith(("TENSORLIB_", "OMP_")) }, sort_keys=True), } + metadata.update(build_metadata(args.executable, args.source_revision)) print("Host:") for key, value in metadata.items(): print(f" {key.replace('_', ' ')}: {value}") @@ -64,6 +98,8 @@ def main(): command += ["--smoke"] if args.smoke else ["--profile", args.profile] if args.threads: command += ["--threads", args.threads] + if args.no_diagnostics: + command += ["--no-diagnostics"] try: result = subprocess.run(command, capture_output=True, text=True, check=False) print(result.stdout, end="") diff --git a/scripts/sweep_matmul.py b/scripts/sweep_matmul.py new file mode 100644 index 0000000..82272de --- /dev/null +++ b/scripts/sweep_matmul.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Build and validate blocking variants, then rotate fresh-process matmul runs.""" + +import argparse +import csv +import itertools +import os +from pathlib import Path +import statistics +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True) + parser.add_argument("--mc", default="32,64,128") + parser.add_argument("--kc", default="64,128,256") + parser.add_argument("--nc", type=int, default=64) + parser.add_argument("--threads", default="1,6") + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + if args.repeats < 3: + parser.error("at least three fresh processes per variant are required") + root = Path(__file__).resolve().parents[1] + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=False) + revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + dirty = subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=no"], + cwd=root, text=True).strip() + if dirty: + parser.error("commit the source before building attributable variants") + variants = list(itertools.product(map(int, args.mc.split(",")), map(int, args.kc.split(",")))) + for mc, kc in variants: + build = output / f"mc{mc}-kc{kc}-nc{args.nc}" + flags = (f"-O3 -g -DNDEBUG -fno-omit-frame-pointer -DTENSORLIB_MATMUL_CONFIG_MC={mc} " + f"-DTENSORLIB_MATMUL_CONFIG_KC={kc} -DTENSORLIB_MATMUL_CONFIG_NC={args.nc}") + with (output / f"build-{mc}-{kc}.log").open("w") as log: + command = ["cmake", "-S", str(root), "-B", str(build), "-DCMAKE_BUILD_TYPE=Release", + "-DTENSORLIB_BUILD_BENCHMARKS=ON", "-DTENSORLIB_ENABLE_OPENMP=ON", + "-DTENSORLIB_WARNINGS_AS_ERRORS=ON", "-DTENSORLIB_NATIVE_OPTIMIZATIONS=ON", + "-DCMAKE_C_FLAGS_RELEASE=" + flags] + subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True) + subprocess.run(["cmake", "--build", str(build), "--parallel", "2", "--target", + "bench_tensorlib", "test_tensor_matmul"], stdout=log, + stderr=subprocess.STDOUT, check=True) + subprocess.run(["ctest", "--test-dir", str(build), "--output-on-failure", "-R", + "test_(tensor_matmul|matmul_policy_)"], stdout=log, + stderr=subprocess.STDOUT, check=True) + print(f"Validated MC={mc} KC={kc} NC={args.nc}", flush=True) + environment = {k: v for k, v in os.environ.items() if not k.startswith(("OMP_", "TENSORLIB_"))} + environment["OMP_DYNAMIC"] = "FALSE" + measurements = {} + for repeat in range(args.repeats): + order = variants[repeat % len(variants):] + variants[:repeat % len(variants)] + if repeat % 2: + order.reverse() + for mc, kc in order: + build = output / f"mc{mc}-kc{kc}-nc{args.nc}" + path = output / f"mc{mc}-kc{kc}-run{repeat}.csv" + command = [sys.executable, str(root / "scripts/run_benchmarks.py"), "--executable", + str(build / "bench_tensorlib"), "--suite", "matmul", "--profile", "quick", + "--threads", args.threads, "--source-revision", revision, "--csv", str(path)] + with path.with_suffix(".log").open("w") as log: + subprocess.run(command, env=environment, stdout=log, + stderr=subprocess.STDOUT, check=True) + with path.open(newline="") as source: + for row in csv.DictReader(source): + if row["status"] != "ok": + raise ValueError("A matmul measurement failed") + key = (mc, kc, row["case"], row["requested_threads"]) + measurements.setdefault(key, []).append(float(row["median_seconds"]) * 1000) + print(f"Measured MC={mc} KC={kc} repeat={repeat + 1}", flush=True) + with (output / "summary.csv").open("w", newline="") as destination: + writer = csv.writer(destination) + writer.writerow(["mc", "kc", "nc", "case", "threads", "median_ms", "min_ms", "max_ms"]) + for (mc, kc, case, threads), samples in sorted(measurements.items()): + writer.writerow([mc, kc, args.nc, case, threads, statistics.median(samples), + min(samples), max(samples)]) + + +if __name__ == "__main__": + main() diff --git a/src/autograd/autograd_backward.c b/src/autograd/autograd_backward.c index 4a63f26..86372cf 100644 --- a/src/autograd/autograd_backward.c +++ b/src/autograd/autograd_backward.c @@ -1,6 +1,7 @@ #define _POSIX_C_SOURCE 200809L #include +#include "../tensor/tensor_alloc_internal.h" #include #include @@ -28,6 +29,18 @@ typedef struct { static int backward_stats_enabled; static ag_backward_stats backward_stats; +static void record_operation_memory(int operation, const tensor_alloc_stats* before) +{ + tensor_alloc_stats after; + tensor_alloc_stats_read(&after); + backward_stats.operation_copied_bytes[operation] += + after.copied_bytes - before->copied_bytes; + size_t bytes = after.allocated_bytes - before->allocated_bytes; + for (int kind = 0; kind < TENSOR_ALLOC_AUX_KINDS; ++kind) + bytes += after.auxiliary_bytes[kind] - before->auxiliary_bytes[kind]; + backward_stats.operation_allocated_bytes[operation] += bytes; +} + static double backward_now(void) { #ifdef _WIN32 @@ -75,8 +88,8 @@ static void sum_contiguous_suffix(const float* source, } #ifdef _OPENMP if (outer_count >= threads * 4 && result_count <= (1 << 14)) { - float* partials = (float*)calloc( - (size_t)threads * (size_t)result_count, sizeof(float)); + float* partials = (float*)tensor_profile_calloc( + (size_t)threads * (size_t)result_count, sizeof(float), TENSOR_ALLOC_GRAPH); if (partials != NULL) { #pragma omp parallel num_threads(threads) { @@ -193,8 +206,8 @@ void ag_backward_stats_record_matmul(int packed_dinput, int fast_dweight) static int append_tensor(tensor_list* list, ag_tensor* value) { if (list->count == list->capacity) { int capacity = list->capacity == 0 ? 16 : list->capacity * 2; - ag_tensor** values = (ag_tensor**)realloc(list->values, - (size_t)capacity * sizeof(*values)); + ag_tensor** values = (ag_tensor**)tensor_profile_realloc(list->values, + (size_t)capacity * sizeof(*values), TENSOR_ALLOC_GRAPH); if (values == NULL) return 1; list->values = values; list->capacity = capacity; @@ -206,8 +219,8 @@ static int append_tensor(tensor_list* list, ag_tensor* value) { static int append_node(node_list* list, ag_node* node) { if (list->count == list->capacity) { int capacity = list->capacity == 0 ? 16 : list->capacity * 2; - ag_node** values = (ag_node**)realloc(list->values, - (size_t)capacity * sizeof(*values)); + ag_node** values = (ag_node**)tensor_profile_realloc(list->values, + (size_t)capacity * sizeof(*values), TENSOR_ALLOC_GRAPH); if (values == NULL) return 1; list->values = values; list->capacity = capacity; @@ -218,8 +231,8 @@ static int append_node(node_list* list, ag_node* node) { static int collect_graph(ag_tensor* value, tensor_list* tensors, node_list* nodes) { if (value == NULL || value->graph_index >= 0) return value == NULL; - value->graph_index = tensors->count; if (append_tensor(tensors, value) != 0) return 1; + value->graph_index = tensors->count - 1; if (value->creator == NULL) return 0; for (int i = 0; i < value->creator->input_count; ++i) { if (collect_graph(value->creator->inputs[i], tensors, nodes) != 0) return 1; @@ -288,7 +301,7 @@ tensor* ag_sum_to_shape(const tensor* source, const tensor* target, float scale) } int* coords = source->ndim > 0 - ? (int*)calloc((size_t)source->ndim, sizeof(*coords)) : NULL; + ? (int*)tensor_profile_calloc((size_t)source->ndim, sizeof(*coords), TENSOR_ALLOC_GRAPH) : NULL; if (backward_stats_enabled) { ++backward_stats.reduction_generic_calls; backward_stats.reduction_generic_elements += @@ -556,7 +569,7 @@ int ag_backward_with_grad_ex(ag_tensor* output, (size_t)contribution_capacity; size_t workspace_bytes = pointer_count * sizeof(tensor*) + (size_t)reduction_coord_capacity * sizeof(int); - workspace = calloc(1, workspace_bytes); + workspace = tensor_profile_calloc(1, workspace_bytes, TENSOR_ALLOC_GRAPH); if (workspace == NULL) goto cleanup; pass_gradients = (tensor**)workspace; merged_gradients = pass_gradients + tensors.count; @@ -573,6 +586,9 @@ int ag_backward_with_grad_ex(ag_tensor* output, ag_node* node = nodes.values[node_index]; int gradient_index = node->output->graph_index; if (gradient_index < 0 || pass_gradients[gradient_index] == NULL) goto cleanup; + int profiling = backward_stats_enabled; + tensor_alloc_stats memory_before; + if (profiling) tensor_alloc_stats_read(&memory_before); if (node->operation == AG_OP_SLICE && node->input_count == 1 && node->inputs[0]->requires_grad) { @@ -582,10 +598,15 @@ int ag_backward_with_grad_ex(ag_tensor* output, ag_accumulate_slice_gradient( node, pass_gradients[gradient_index], &pass_gradients[destination]) == 0) { - if (backward_stats_enabled) { + if (profiling) { backward_stats.operation_seconds[AG_OP_SLICE] += backward_elapsed(started); ++backward_stats.operation_calls[AG_OP_SLICE]; + record_operation_memory(AG_OP_SLICE, &memory_before); + } + if (options->retention == AG_GRAD_RETAIN_LEAVES) { + t_free(pass_gradients[gradient_index]); + pass_gradients[gradient_index] = NULL; } continue; } @@ -598,12 +619,13 @@ int ag_backward_with_grad_ex(ag_tensor* output, started = backward_stats_enabled ? backward_now() : 0.0; int backward_status = node->backward(node, pass_gradients[gradient_index], contributions); - if (backward_stats_enabled) { + if (profiling) { int operation = (int)node->operation; if (operation >= 0 && operation < AG_BACKWARD_OP_COUNT) { backward_stats.operation_seconds[operation] += backward_elapsed(started); ++backward_stats.operation_calls[operation]; + record_operation_memory(operation, &memory_before); } } if (backward_status != 0) { @@ -630,6 +652,12 @@ int ag_backward_with_grad_ex(ag_tensor* output, } contributions[input_index] = NULL; } + /* Reverse topological order has consumed this intermediate gradient. + * Shared tensor storage remains alive through any propagated views. */ + if (options->retention == AG_GRAD_RETAIN_LEAVES) { + t_free(pass_gradients[gradient_index]); + pass_gradients[gradient_index] = NULL; + } } started = backward_stats_enabled ? backward_now() : 0.0; diff --git a/src/autograd/autograd_core.c b/src/autograd/autograd_core.c index cd43359..ab07270 100644 --- a/src/autograd/autograd_core.c +++ b/src/autograd/autograd_core.c @@ -1,4 +1,5 @@ #include +#include "../tensor/tensor_alloc_internal.h" #include #include "../../include/tensorlib/autograd.h" @@ -13,7 +14,7 @@ ag_tensor* ag_from_owned_tensor(tensor* value, int requires_grad) { return NULL; } - ag_tensor* result = (ag_tensor*)calloc(1, sizeof(*result)); + ag_tensor* result = (ag_tensor*)tensor_profile_calloc(1, sizeof(*result), TENSOR_ALLOC_GRAPH); if (result == NULL) { t_free(value); return NULL; @@ -29,7 +30,7 @@ ag_tensor* ag_from_owned_tensor(tensor* value, int requires_grad) { ag_tensor* ag_detach(const ag_tensor* value) { if (value == NULL || !tensor_has_valid_metadata(value->value)) return NULL; - tensor* alias = (tensor*)calloc(1, sizeof(*alias)); + tensor* alias = (tensor*)tensor_profile_calloc(1, sizeof(*alias), TENSOR_ALLOC_GRAPH); if (alias == NULL) return NULL; alias->ndim = value->value->ndim; alias->offset = value->value->offset; @@ -114,21 +115,21 @@ ag_tensor* ag_make_result(tensor* output, return result; } - ag_node* node = (ag_node*)calloc(1, sizeof(*node)); + ag_node* node = (ag_node*)tensor_profile_calloc(1, sizeof(*node), TENSOR_ALLOC_GRAPH); if (node == NULL) { if (free_context != NULL) free_context(context); ag_tensor_release(result); return NULL; } - node->inputs = (ag_tensor**)calloc((size_t)input_count, sizeof(*node->inputs)); + node->inputs = (ag_tensor**)tensor_profile_calloc((size_t)input_count, sizeof(*node->inputs), TENSOR_ALLOC_GRAPH); if (node->inputs == NULL) { free(node); if (free_context != NULL) free_context(context); ag_tensor_release(result); return NULL; } - node->input_versions = (uint64_t*)calloc((size_t)input_count, - sizeof(*node->input_versions)); + node->input_versions = (uint64_t*)tensor_profile_calloc((size_t)input_count, + sizeof(*node->input_versions), TENSOR_ALLOC_GRAPH); if (node->input_versions == NULL) { free(node->inputs); free(node); diff --git a/src/autograd/autograd_cross_entropy.c b/src/autograd/autograd_cross_entropy.c index 642dc5e..5485e41 100644 --- a/src/autograd/autograd_cross_entropy.c +++ b/src/autograd/autograd_cross_entropy.c @@ -87,12 +87,12 @@ ag_tensor* ag_cross_entropy(const ag_tensor* logits, const tensor* targets) } classes = logits->value->dims[logits->value->ndim - 1]; rows = tensor_numel((tensor*)targets); - context = (cross_entropy_context*)calloc(1, sizeof(*context)); + context = (cross_entropy_context*)tensor_profile_calloc(1, sizeof(*context), TENSOR_ALLOC_GRAPH); output = t_alloc(0, NULL); if (context == NULL || output == NULL) goto fail; context->rows = rows; context->classes = classes; - context->targets = (int*)malloc((size_t)rows * sizeof(*context->targets)); + context->targets = (int*)tensor_profile_malloc((size_t)rows * sizeof(*context->targets), TENSOR_ALLOC_GRAPH); if (context->targets == NULL) goto fail; for (int row = 0; row < rows; ++row) { float target = targets->storage->data[tensor_flat_index(targets, row)]; diff --git a/src/autograd/autograd_gather.c b/src/autograd/autograd_gather.c index 04261f8..796df9c 100644 --- a/src/autograd/autograd_gather.c +++ b/src/autograd/autograd_gather.c @@ -65,7 +65,7 @@ ag_tensor* ag_gather_rows(const ag_tensor* table, const tensor* indices) } output = t_gather_rows(table->value, (tensor*)indices); if (output == NULL) return NULL; - context = (gather_context*)calloc(1, sizeof(*context)); + context = (gather_context*)tensor_profile_calloc(1, sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; diff --git a/src/autograd/autograd_layer_norm.c b/src/autograd/autograd_layer_norm.c index 0b080a8..cb9b8c8 100644 --- a/src/autograd/autograd_layer_norm.c +++ b/src/autograd/autograd_layer_norm.c @@ -72,13 +72,13 @@ static int backward_layer_norm(const ag_node* node, (void)threads; #endif if (threads > 1 && weight_gradient != NULL) { - weight_partials = (float*)calloc((size_t)threads * (size_t)width, - sizeof(*weight_partials)); + weight_partials = (float*)tensor_profile_calloc((size_t)threads * (size_t)width, + sizeof(*weight_partials), TENSOR_ALLOC_GRAPH); if (weight_partials == NULL) goto fail; } if (threads > 1 && bias_gradient != NULL) { - bias_partials = (float*)calloc((size_t)threads * (size_t)width, - sizeof(*bias_partials)); + bias_partials = (float*)tensor_profile_calloc((size_t)threads * (size_t)width, + sizeof(*bias_partials), TENSOR_ALLOC_GRAPH); if (bias_partials == NULL) goto fail; } @@ -186,12 +186,12 @@ ag_tensor* ag_layer_norm(const ag_tensor* input, return NULL; } output = t_alloc(input->value->ndim, input->value->dims); - context = (layer_norm_context*)calloc(1, sizeof(*context)); + context = (layer_norm_context*)tensor_profile_calloc(1, sizeof(*context), TENSOR_ALLOC_GRAPH); if (output == NULL || context == NULL) goto fail; context->width = width; context->rows = tensor_numel(input->value) / width; - context->means = (float*)calloc((size_t)context->rows, sizeof(float)); - context->inverse_stds = (float*)calloc((size_t)context->rows, sizeof(float)); + context->means = (float*)tensor_profile_calloc((size_t)context->rows, sizeof(float), TENSOR_ALLOC_GRAPH); + context->inverse_stds = (float*)tensor_profile_calloc((size_t)context->rows, sizeof(float), TENSOR_ALLOC_GRAPH); if (context->means == NULL || context->inverse_stds == NULL) goto fail; for (int row = 0; row < context->rows; ++row) { diff --git a/src/autograd/autograd_matmul.c b/src/autograd/autograd_matmul.c index f0da6e9..b47230b 100644 --- a/src/autograd/autograd_matmul.c +++ b/src/autograd/autograd_matmul.c @@ -1,4 +1,5 @@ #include +#include "../tensor/tensor_alloc_internal.h" #include "./../../include/tensorlib/autograd_internal.h" #include "../tensor/tensor_matmul_internal.h" @@ -126,7 +127,7 @@ ag_tensor* ag_matmul_packed_rhs_with_backward_pack( matmul_context* context = NULL; if (a == NULL || b == NULL || packed_rhs == NULL) return NULL; if (backward_rhs != NULL) { - context = (matmul_context*)calloc(1, sizeof(*context)); + context = (matmul_context*)tensor_profile_calloc(1, sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) return NULL; context->backward_rhs = (tensor_matmul_packed_rhs*)backward_rhs; t_retain_matmul_packed_rhs(context->backward_rhs); diff --git a/src/autograd/autograd_ops.c b/src/autograd/autograd_ops.c index 7983652..bf7f2da 100644 --- a/src/autograd/autograd_ops.c +++ b/src/autograd/autograd_ops.c @@ -1,5 +1,6 @@ #include #include +#include "../tensor/tensor_alloc_internal.h" #include "./../../include/tensorlib/autograd_internal.h" #include "../tensor/parallel.h" @@ -166,7 +167,7 @@ static ag_tensor* apply_scalar(const ag_tensor* value, if (value == NULL || forward == NULL || backward == NULL) return NULL; output = forward(value->value, scalar); if (output == NULL) return NULL; - context = (scalar_context*)malloc(sizeof(*context)); + context = (scalar_context*)tensor_profile_malloc(sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; @@ -274,7 +275,7 @@ ag_tensor* ag_pow(const ag_tensor* value, float exponent) { if (value == NULL) return NULL; tensor* output = t_pow(value->value, exponent); if (output == NULL) return NULL; - pow_context* context = (pow_context*)malloc(sizeof(*context)); + pow_context* context = (pow_context*)tensor_profile_malloc(sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; @@ -433,7 +434,7 @@ static int backward_gelu(const ag_node* node, } } else { int* coords = input->ndim > 0 - ? (int*)calloc((size_t)input->ndim, sizeof(*coords)) : NULL; + ? (int*)tensor_profile_calloc((size_t)input->ndim, sizeof(*coords), TENSOR_ALLOC_GRAPH) : NULL; if (input->ndim > 0 && coords == NULL) { t_free(result); return 1; diff --git a/src/autograd/autograd_reduc.c b/src/autograd/autograd_reduc.c index b221deb..9f19aa5 100644 --- a/src/autograd/autograd_reduc.c +++ b/src/autograd/autograd_reduc.c @@ -1,5 +1,6 @@ #include #include +#include "../tensor/tensor_alloc_internal.h" #include "./../../include/tensorlib/autograd_internal.h" @@ -70,8 +71,8 @@ static int backward_max(const ag_node* node, tensor* output = node->output->value; tensor* gradient = ag_full_like(input, 0.0f); int* output_coords = output->ndim > 0 - ? (int*)calloc((size_t)output->ndim, sizeof(int)) : NULL; - int* input_coords = (int*)calloc((size_t)input->ndim, sizeof(int)); + ? (int*)tensor_profile_calloc((size_t)output->ndim, sizeof(int), TENSOR_ALLOC_GRAPH) : NULL; + int* input_coords = (int*)tensor_profile_calloc((size_t)input->ndim, sizeof(int), TENSOR_ALLOC_GRAPH); if (gradient == NULL || (output->ndim > 0 && output_coords == NULL) || input_coords == NULL) { t_free(gradient); free(output_coords); free(input_coords); return 1; @@ -124,7 +125,7 @@ static ag_tensor* apply_reduction(const ag_tensor* value, if (value == NULL) return NULL; tensor* output = keepdim ? keep_dim(value->value, dim) : remove_dim(value->value, dim); if (output == NULL) return NULL; - reduction_context* context = (reduction_context*)malloc(sizeof(*context)); + reduction_context* context = (reduction_context*)tensor_profile_malloc(sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; } context->dim = dim; context->keepdim = keepdim != 0; diff --git a/src/autograd/autograd_softmax.c b/src/autograd/autograd_softmax.c index 88dbd76..c5a9e7b 100644 --- a/src/autograd/autograd_softmax.c +++ b/src/autograd/autograd_softmax.c @@ -92,7 +92,7 @@ ag_tensor* ag_softmax_last_dim(const ag_tensor* input, if (causal && (input->value->ndim < 2 || input->value->dims[input->value->ndim - 2] != width)) return NULL; output = t_alloc(input->value->ndim, input->value->dims); - context = (softmax_context*)calloc(1, sizeof(*context)); + context = (softmax_context*)tensor_profile_calloc(1, sizeof(*context), TENSOR_ALLOC_GRAPH); if (output == NULL || context == NULL) goto fail; context->width = width; context->rows = tensor_numel(input->value) / width; diff --git a/src/autograd/autograd_view.c b/src/autograd/autograd_view.c index 0734d35..6439e19 100644 --- a/src/autograd/autograd_view.c +++ b/src/autograd/autograd_view.c @@ -1,4 +1,5 @@ #include +#include "../tensor/tensor_alloc_internal.h" #include #include "./../../include/tensorlib/autograd_internal.h" @@ -71,8 +72,8 @@ static int backward_slice(const ag_node* node, input_gradients[0] = gradient; return 0; } - int* output_coords = (int*)calloc((size_t)output_gradient->ndim, sizeof(int)); - int* input_coords = (int*)calloc((size_t)input->ndim, sizeof(int)); + int* output_coords = (int*)tensor_profile_calloc((size_t)output_gradient->ndim, sizeof(int), TENSOR_ALLOC_GRAPH); + int* input_coords = (int*)tensor_profile_calloc((size_t)input->ndim, sizeof(int), TENSOR_ALLOC_GRAPH); if (output_coords == NULL || input_coords == NULL) { t_free(gradient); free(output_coords); free(input_coords); return 1; @@ -176,7 +177,7 @@ ag_tensor* ag_transpose(const ag_tensor* value, int dim0, int dim1) { if (value == NULL) return NULL; tensor* output = t_transpose(value->value, dim0, dim1); if (output == NULL) return NULL; - transpose_context* context = (transpose_context*)malloc(sizeof(*context)); + transpose_context* context = (transpose_context*)tensor_profile_malloc(sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; } context->dim0 = dim0; context->dim1 = dim1; @@ -189,7 +190,7 @@ ag_tensor* ag_slice(const ag_tensor* value, int dim, int start, int end) { if (value == NULL) return NULL; tensor* output = t_slice(value->value, dim, start, end); if (output == NULL) return NULL; - slice_context* context = (slice_context*)malloc(sizeof(*context)); + slice_context* context = (slice_context*)tensor_profile_malloc(sizeof(*context), TENSOR_ALLOC_GRAPH); if (context == NULL) { t_free(output); return NULL; } context->dim = dim; context->start = start; diff --git a/src/tensor/tensor_alloc.c b/src/tensor/tensor_alloc.c index c03f74b..837d215 100644 --- a/src/tensor/tensor_alloc.c +++ b/src/tensor/tensor_alloc.c @@ -12,6 +12,52 @@ static volatile unsigned long long stats_frees; static volatile size_t stats_allocated_bytes; static volatile size_t stats_live_bytes; static volatile size_t stats_peak_live_bytes; +static volatile unsigned long long stats_aux_allocations[TENSOR_ALLOC_AUX_KINDS]; +static volatile size_t stats_aux_bytes[TENSOR_ALLOC_AUX_KINDS]; +static volatile size_t stats_copied_bytes; + +void tensor_alloc_record_auxiliary(int kind, size_t bytes) +{ + if (!stats_enabled || kind < 0 || kind >= TENSOR_ALLOC_AUX_KINDS) return; +#ifdef _OPENMP +#pragma omp atomic update +#endif + ++stats_aux_allocations[kind]; +#ifdef _OPENMP +#pragma omp atomic update +#endif + stats_aux_bytes[kind] += bytes; +} + +void tensor_alloc_record_copy(size_t bytes) +{ + if (!stats_enabled) return; +#ifdef _OPENMP +#pragma omp atomic update +#endif + stats_copied_bytes += bytes; +} + +void* tensor_profile_malloc(size_t bytes, int kind) +{ + void* result = malloc(bytes); + if (result != NULL) tensor_alloc_record_auxiliary(kind, bytes); + return result; +} + +void* tensor_profile_calloc(size_t count, size_t bytes, int kind) +{ + void* result = calloc(count, bytes); + if (result != NULL) tensor_alloc_record_auxiliary(kind, count * bytes); + return result; +} + +void* tensor_profile_realloc(void* pointer, size_t bytes, int kind) +{ + void* result = realloc(pointer, bytes); + if (result != NULL) tensor_alloc_record_auxiliary(kind, bytes); + return result; +} static void record_storage_alloc(size_t bytes) { @@ -61,6 +107,11 @@ void tensor_alloc_stats_reset(void) stats_allocated_bytes = 0; stats_live_bytes = 0; stats_peak_live_bytes = 0; + for (int kind = 0; kind < TENSOR_ALLOC_AUX_KINDS; ++kind) { + stats_aux_allocations[kind] = 0; + stats_aux_bytes[kind] = 0; + } + stats_copied_bytes = 0; } void tensor_alloc_stats_reset_counters(void) @@ -69,6 +120,11 @@ void tensor_alloc_stats_reset_counters(void) stats_frees = 0; stats_allocated_bytes = 0; stats_peak_live_bytes = stats_live_bytes; + for (int kind = 0; kind < TENSOR_ALLOC_AUX_KINDS; ++kind) { + stats_aux_allocations[kind] = 0; + stats_aux_bytes[kind] = 0; + } + stats_copied_bytes = 0; } void tensor_alloc_stats_read(tensor_alloc_stats* result) @@ -79,6 +135,11 @@ void tensor_alloc_stats_read(tensor_alloc_stats* result) result->allocated_bytes = stats_allocated_bytes; result->live_bytes = stats_live_bytes; result->peak_live_bytes = stats_peak_live_bytes; + for (int kind = 0; kind < TENSOR_ALLOC_AUX_KINDS; ++kind) { + result->auxiliary_allocations[kind] = stats_aux_allocations[kind]; + result->auxiliary_bytes[kind] = stats_aux_bytes[kind]; + } + result->copied_bytes = stats_copied_bytes; } void add_ref_count(Storage* a, tensor* b) { @@ -92,7 +153,7 @@ Storage* s_alloc(int ndim, const int* dims) { size_t count; if (!tensor_checked_numel(ndim, dims, &count)) return NULL; - Storage* s = (Storage*)malloc(sizeof(Storage)); + Storage* s = (Storage*)tensor_profile_malloc(sizeof(Storage), TENSOR_ALLOC_METADATA); if (s == NULL) return NULL; s->ref_count = 1; s->size = (int)count; @@ -111,12 +172,12 @@ tensor* t_alloc(int ndim, const int* dims) { if (!tensor_checked_numel(ndim, dims, &count)) return NULL; (void)count; - tensor* a = (tensor*)calloc(1, sizeof(tensor)); + tensor* a = (tensor*)tensor_profile_calloc(1, sizeof(tensor), TENSOR_ALLOC_METADATA); if (a == NULL) return NULL; a->ndim = ndim; if (ndim > 0) { - int* strides = (int*)malloc((size_t)ndim * sizeof(int)); + int* strides = (int*)tensor_profile_malloc((size_t)ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (strides == NULL) { t_free(a); return NULL; @@ -165,7 +226,7 @@ int init_t(tensor* c, tensor* ref) { c->ndim = ref->ndim; c->offset = 0; - c->storage = (Storage*)malloc(sizeof(Storage)); + c->storage = (Storage*)tensor_profile_malloc(sizeof(Storage), TENSOR_ALLOC_METADATA); if (c->storage == NULL) return 1; c->storage->ref_count = 1; c->storage->size = total_elements; @@ -179,7 +240,7 @@ int init_t(tensor* c, tensor* ref) { record_storage_alloc((size_t)total_elements * sizeof(float)); if (ref->ndim > 0) { - int* strides = (int*)malloc((size_t)ref->ndim * sizeof(int)); + int* strides = (int*)tensor_profile_malloc((size_t)ref->ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (strides == NULL) { free(c->storage->data); free(c->storage); @@ -229,6 +290,7 @@ tensor* t_clone(tensor* t) { #else memcpy(destination, source, (size_t)total_elements * sizeof(float)); #endif + tensor_alloc_record_copy((size_t)total_elements * sizeof(float)); return a; } @@ -258,6 +320,7 @@ tensor* t_clone(tensor* t) { t->storage->data + source, (size_t)inner * sizeof(float)); } + tensor_alloc_record_copy((size_t)total_elements * sizeof(float)); return a; } @@ -292,10 +355,11 @@ tensor* t_clone(tensor* t) { } } } + tensor_alloc_record_copy((size_t)total_elements * sizeof(float)); return a; } - int* coords = (int*)calloc((size_t)t->ndim, sizeof(int)); + int* coords = (int*)tensor_profile_calloc((size_t)t->ndim, sizeof(int), TENSOR_ALLOC_METADATA); int src_idx = t->offset; if (t->ndim > 0 && coords == NULL) { t_free(a); @@ -313,6 +377,7 @@ tensor* t_clone(tensor* t) { } } free(coords); + tensor_alloc_record_copy((size_t)total_elements * sizeof(float)); return a; } diff --git a/src/tensor/tensor_alloc_internal.h b/src/tensor/tensor_alloc_internal.h index b338187..ee95989 100644 --- a/src/tensor/tensor_alloc_internal.h +++ b/src/tensor/tensor_alloc_internal.h @@ -6,18 +6,29 @@ #include "../../include/tensorlib/tensor.h" +enum { TENSOR_ALLOC_METADATA, TENSOR_ALLOC_GRAPH, TENSOR_ALLOC_MATMUL, + TENSOR_ALLOC_AUX_KINDS }; + typedef struct { uint64_t allocations; uint64_t frees; size_t allocated_bytes; size_t live_bytes; size_t peak_live_bytes; + uint64_t auxiliary_allocations[TENSOR_ALLOC_AUX_KINDS]; + size_t auxiliary_bytes[TENSOR_ALLOC_AUX_KINDS]; + size_t copied_bytes; } tensor_alloc_stats; void tensor_alloc_stats_enable(int enabled); void tensor_alloc_stats_reset(void); void tensor_alloc_stats_reset_counters(void); void tensor_alloc_stats_read(tensor_alloc_stats* result); +void tensor_alloc_record_auxiliary(int kind, size_t bytes); +void tensor_alloc_record_copy(size_t bytes); +void* tensor_profile_malloc(size_t bytes, int kind); +void* tensor_profile_calloc(size_t count, size_t bytes, int kind); +void* tensor_profile_realloc(void* pointer, size_t bytes, int kind); /* Shared private indexing helpers used by tensor, autograd, and NN code. */ int tensor_flat_index(const tensor* value, int flat); diff --git a/src/tensor/tensor_core.c b/src/tensor/tensor_core.c index 783df8b..f60472b 100644 --- a/src/tensor/tensor_core.c +++ b/src/tensor/tensor_core.c @@ -94,8 +94,8 @@ int tensor_copy_metadata(int ndim, const int* dims, const int* strides, int** ou if (dims == NULL || strides == NULL) return 1; size_t metadata_bytes = (size_t)ndim * sizeof(int); - *out_dims = (int*)malloc(metadata_bytes); - *out_strides = (int*)malloc(metadata_bytes); + *out_dims = (int*)tensor_profile_malloc(metadata_bytes, TENSOR_ALLOC_METADATA); + *out_strides = (int*)tensor_profile_malloc(metadata_bytes, TENSOR_ALLOC_METADATA); if (*out_dims == NULL || *out_strides == NULL) { free(*out_dims); free(*out_strides); diff --git a/src/tensor/tensor_matmul.c b/src/tensor/tensor_matmul.c index 5bf73ae..4986076 100644 --- a/src/tensor/tensor_matmul.c +++ b/src/tensor/tensor_matmul.c @@ -2,6 +2,7 @@ #include #include #include +#include "tensor_alloc_internal.h" #include #if defined(_WIN32) #include @@ -23,11 +24,12 @@ #include "../../include/tensorlib/tensor_matmul.h" #include "parallel.h" #include "tensor_matmul_internal.h" +#include "tensor_matmul_tuning.h" enum { - TENSORLIB_MATMUL_MC = 64, - TENSORLIB_MATMUL_NC = 64, - TENSORLIB_MATMUL_KC = 128, + TENSORLIB_MATMUL_MC = TENSORLIB_MATMUL_CONFIG_MC, + TENSORLIB_MATMUL_NC = TENSORLIB_MATMUL_CONFIG_NC, + TENSORLIB_MATMUL_KC = TENSORLIB_MATMUL_CONFIG_KC, TENSORLIB_MATMUL_NR = 16, TENSORLIB_MATMUL_MR = 4, TENSORLIB_MATMUL_MAX_NDIM = 32 @@ -70,9 +72,11 @@ static int matmul_2d_packed_rhs_avx2(const tensor* lhs, int lhs_base, static void* matmul_aligned_malloc(size_t bytes) { #if defined(_WIN32) - return _aligned_malloc(bytes, 32); + void* result = _aligned_malloc(bytes, 32); + if (result != NULL) tensor_alloc_record_auxiliary(TENSOR_ALLOC_MATMUL, bytes); + return result; #else - return malloc(bytes); + return tensor_profile_malloc(bytes, TENSOR_ALLOC_MATMUL); #endif } @@ -129,7 +133,7 @@ tensor_matmul_packed_rhs* t_pack_matmul_rhs(const tensor* rhs) { if (inner_stride <= 0 || column_stride <= 0) return NULL; tensor_matmul_packed_rhs* packed = - (tensor_matmul_packed_rhs*)calloc(1, sizeof(*packed)); + (tensor_matmul_packed_rhs*)tensor_profile_calloc(1, sizeof(*packed), TENSOR_ALLOC_MATMUL); if (packed == NULL) return NULL; packed->ref_count = 1; @@ -145,7 +149,7 @@ tensor_matmul_packed_rhs* t_pack_matmul_rhs(const tensor* rhs) { } if (batch_rank > 0) { - packed->batch_dims = (int*)malloc((size_t)batch_rank * sizeof(int)); + packed->batch_dims = (int*)tensor_profile_malloc((size_t)batch_rank * sizeof(int), TENSOR_ALLOC_MATMUL); if (packed->batch_dims == NULL) { t_free_matmul_packed_rhs(packed); return NULL; @@ -367,7 +371,7 @@ tensor* t_matmul_packed_rhs(const tensor* lhs, ? lhs_info.batch_rank : rhs->batch_rank; int* batch_dims = NULL; if (batch_ndim > 0) { - batch_dims = (int*)malloc((size_t)batch_ndim * sizeof(int)); + batch_dims = (int*)tensor_profile_malloc((size_t)batch_ndim * sizeof(int), TENSOR_ALLOC_MATMUL); if (batch_dims == NULL) return NULL; } @@ -382,7 +386,7 @@ tensor* t_matmul_packed_rhs(const tensor* lhs, } int output_dims_count = batch_ndim + 2; - int* output_dims = (int*)malloc((size_t)output_dims_count * sizeof(int)); + int* output_dims = (int*)tensor_profile_malloc((size_t)output_dims_count * sizeof(int), TENSOR_ALLOC_MATMUL); if (output_dims == NULL) { free(batch_dims); return NULL; @@ -426,8 +430,11 @@ tensor* t_matmul_packed_rhs(const tensor* lhs, batch_ndim <= TENSORLIB_MATMUL_MAX_NDIM ? task_count : 1); size_t workspace_values = (size_t)TENSORLIB_MATMUL_MC * (size_t)TENSORLIB_MATMUL_KC; - float* workspaces = (float*)matmul_aligned_malloc( - (size_t)threads * workspace_values * sizeof(float)); + size_t workspace_bytes; + int workspace_valid = checked_size_multiply( + (size_t)threads, workspace_values * sizeof(float), &workspace_bytes); + float* workspaces = workspace_valid + ? (float*)matmul_aligned_malloc(workspace_bytes) : NULL; if (threads > 1) { #ifdef _OPENMP @@ -1062,6 +1069,7 @@ static tensor* try_contiguous_projected_rhs_gradient( memcpy(result->storage->data + (size_t)projection * (size_t)inner * (size_t)columns, matrix->storage->data + matrix->offset, matrix_bytes); + tensor_alloc_record_copy(matrix_bytes); t_free(matrix); t_free(gradient_matrix); t_free(gradient_slice); } t_free(lhs_transpose); t_free(lhs_matrix); @@ -1344,7 +1352,7 @@ tensor* t_matmul(tensor* a, tensor* b) { : b_info.batch_rank; int* batch_dims = NULL; if (batch_ndim > 0) { - batch_dims = (int*)malloc((size_t)batch_ndim * sizeof(int)); + batch_dims = (int*)tensor_profile_malloc((size_t)batch_ndim * sizeof(int), TENSOR_ALLOC_MATMUL); if (batch_dims == NULL) return NULL; } @@ -1363,7 +1371,7 @@ tensor* t_matmul(tensor* a, tensor* b) { int output_ndim = batch_ndim + 2 - a_info.is_vector - b_info.is_vector; int* output_dims = NULL; if (output_ndim > 0) { - output_dims = (int*)malloc((size_t)output_ndim * sizeof(int)); + output_dims = (int*)tensor_profile_malloc((size_t)output_ndim * sizeof(int), TENSOR_ALLOC_MATMUL); if (output_dims == NULL) { free(batch_dims); return NULL; diff --git a/src/tensor/tensor_matmul_tuning.h b/src/tensor/tensor_matmul_tuning.h new file mode 100644 index 0000000..0c9d793 --- /dev/null +++ b/src/tensor/tensor_matmul_tuning.h @@ -0,0 +1,27 @@ +#ifndef TENSORLIB_TENSOR_MATMUL_TUNING_H +#define TENSORLIB_TENSOR_MATMUL_TUNING_H + +/* Private experiment controls. Defaults stay portable; MR/NR match the kernel. */ +#ifndef TENSORLIB_MATMUL_CONFIG_MC +#define TENSORLIB_MATMUL_CONFIG_MC 64 +#endif +#ifndef TENSORLIB_MATMUL_CONFIG_NC +#define TENSORLIB_MATMUL_CONFIG_NC 64 +#endif +#ifndef TENSORLIB_MATMUL_CONFIG_KC +#define TENSORLIB_MATMUL_CONFIG_KC 128 +#endif + +#if TENSORLIB_MATMUL_CONFIG_MC < 4 || TENSORLIB_MATMUL_CONFIG_MC > 512 || \ + TENSORLIB_MATMUL_CONFIG_MC % 4 != 0 +#error "Matmul MC must be a multiple of 4 in [4,512]" +#endif +#if TENSORLIB_MATMUL_CONFIG_NC < 16 || TENSORLIB_MATMUL_CONFIG_NC > 512 || \ + TENSORLIB_MATMUL_CONFIG_NC % 16 != 0 +#error "Matmul NC must be a multiple of 16 in [16,512]" +#endif +#if TENSORLIB_MATMUL_CONFIG_KC < 1 || TENSORLIB_MATMUL_CONFIG_KC > 512 +#error "Matmul KC must be in [1,512]" +#endif + +#endif diff --git a/src/tensor/tensor_view.c b/src/tensor/tensor_view.c index d3d249c..e66ab74 100644 --- a/src/tensor/tensor_view.c +++ b/src/tensor/tensor_view.c @@ -1,6 +1,7 @@ #include #include #include +#include "tensor_alloc_internal.h" #include "../../include/tensorlib/tensor.h" static tensor* make_view(tensor* base, int ndim, const int* dims, const int* strides, int offset) { @@ -9,7 +10,7 @@ static tensor* make_view(tensor* base, int ndim, const int* dims, const int* str if (!tensor_checked_numel(ndim, dims, &unused)) return NULL; if (ndim > 0 && strides == NULL) return NULL; - tensor* view = (tensor*)calloc(1, sizeof(tensor)); + tensor* view = (tensor*)tensor_profile_calloc(1, sizeof(tensor), TENSOR_ALLOC_METADATA); if (view == NULL) return NULL; view->ndim = ndim; @@ -72,7 +73,7 @@ tensor* t_reshape(tensor* a, int new_ndim, int* new_dims) { int* new_strides = NULL; if (new_ndim > 0) { - new_strides = (int*)malloc((size_t)new_ndim * sizeof(int)); + new_strides = (int*)tensor_profile_malloc((size_t)new_ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (new_strides == NULL) return NULL; calc_strides(new_ndim, new_dims, new_strides); } @@ -118,7 +119,7 @@ tensor* t_slice(tensor* a, int dim, int start, int end) { return NULL; } - int* new_dims = (int*)malloc((size_t)a->ndim * sizeof(int)); + int* new_dims = (int*)tensor_profile_malloc((size_t)a->ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (new_dims == NULL) return NULL; for (int i = 0; i < a->ndim; i++) { @@ -140,12 +141,12 @@ tensor* t_squeeze(tensor* a, int dim) { return NULL; } - int* new_dims = (int*)malloc((size_t)(a->ndim - 1) * sizeof(int)); + int* new_dims = (int*)tensor_profile_malloc((size_t)(a->ndim - 1) * sizeof(int), TENSOR_ALLOC_METADATA); if (new_dims == NULL) return NULL; int new_ndim = a->ndim - 1; int j = 0; - int* new_strides = (int*)malloc((size_t)new_ndim * sizeof(int)); + int* new_strides = (int*)tensor_profile_malloc((size_t)new_ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (new_strides == NULL) { free(new_dims); return NULL; @@ -173,8 +174,8 @@ tensor* t_unsqueeze(tensor* a, int dim) { } int new_ndim = a->ndim + 1; - int* new_dims = (int*)malloc((size_t)new_ndim * sizeof(int)); - int* new_strides = (int*)malloc((size_t)new_ndim * sizeof(int)); + int* new_dims = (int*)tensor_profile_malloc((size_t)new_ndim * sizeof(int), TENSOR_ALLOC_METADATA); + int* new_strides = (int*)tensor_profile_malloc((size_t)new_ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (new_dims == NULL || new_strides == NULL) { free(new_dims); free(new_strides); @@ -223,7 +224,7 @@ tensor* t_expand(tensor* a, int new_ndim, const int* new_dims) { int* new_strides = NULL; if (new_ndim > 0) { - new_strides = (int*)malloc((size_t)new_ndim * sizeof(int)); + new_strides = (int*)tensor_profile_malloc((size_t)new_ndim * sizeof(int), TENSOR_ALLOC_METADATA); if (new_strides == NULL) return NULL; } diff --git a/tests/test_training_comparison.py b/tests/test_training_comparison.py new file mode 100644 index 0000000..e09a82e --- /dev/null +++ b/tests/test_training_comparison.py @@ -0,0 +1,57 @@ +"""Run with python3 tests/test_training_comparison.py [bench_tensorlib].""" + +import csv +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("comparison", ROOT / "scripts/compare_training.py") +comparison = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(comparison) +EXECUTABLE = sys.argv.pop(1) if len(sys.argv) > 1 else None + + +class TrainingComparisonTests(unittest.TestCase): + def test_paired_ratios_and_invalid_sequences(self): + key = ("nn", "model", "shape", "layout", "fixed", "1") + before = {key: {"median_seconds": "1", "iterations": "1", "checksum": "2"}} + after = {key: {"median_seconds": ".8", "iterations": "1", "checksum": "2"}} + row = comparison.summarize([(before, after)] * 3)[0] + self.assertAlmostEqual(row["change_percent"], -20) + for bad in ({}, {key: dict(after[key], iterations="2")}, + {key: dict(after[key], checksum="9")}, + {key: dict(after[key], cpu="different CPU")}): + with self.assertRaises(ValueError): + comparison.summarize([(before, bad)] * 3) + changed = {key: dict(after[key], executable_sha256="different binary")} + with self.assertRaises(ValueError): + comparison.summarize([(before, after), (before, changed), (before, after)]) + + @unittest.skipUnless(EXECUTABLE, "benchmark executable was not supplied") + def test_training_thread_inventory_and_diagnostic_boundary(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "results.csv" + for diagnostics in (False, True): + command = [EXECUTABLE, "--suite", "training", "--smoke", + "--threads", "1,2", "--csv", str(path)] + if not diagnostics: + command.append("--no-diagnostics") + subprocess.run(command, check=True, stdout=subprocess.DEVNULL) + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + training = [r for r in rows if r["case"].endswith("_train_step")] + keys = [(r["case"], r["requested_threads"]) for r in training] + self.assertEqual(len(keys), 6) + self.assertEqual(len(set(keys)), 6) + self.assertEqual({r["requested_threads"] for r in training}, {"1", "2"}) + self.assertEqual(any(r["suite"] == "nn_phase" for r in rows), diagnostics) + if not diagnostics: + self.assertEqual(len(comparison.read_rows(path)), 6) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/autograd/test_autograd_allocation_failure.c b/tests/unit/autograd/test_autograd_allocation_failure.c new file mode 100644 index 0000000..4d73772 --- /dev/null +++ b/tests/unit/autograd/test_autograd_allocation_failure.c @@ -0,0 +1,98 @@ +#include +#include +#include +#include "../../../include/tensorlib/autograd.h" +#include "../../../src/tensor/tensor_alloc_internal.h" + +void* __real_malloc(size_t bytes); +void* __real_calloc(size_t count, size_t bytes); +void* __real_realloc(void* pointer, size_t bytes); +static int remaining = -1; +static int failures; + +static int fail_now(void) { + if (remaining < 0) return 0; + if (remaining-- != 0) return 0; + remaining = -1; + ++failures; + return 1; +} + +void* __wrap_malloc(size_t bytes) { + return fail_now() ? NULL : __real_malloc(bytes); +} +void* __wrap_calloc(size_t count, size_t bytes) { + return fail_now() ? NULL : __real_calloc(count, bytes); +} +void* __wrap_realloc(void* pointer, size_t bytes) { + return fail_now() ? NULL : __real_realloc(pointer, bytes); +} + +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "allocation rollback check failed at line %d\n", __LINE__); \ + return 1; } } while (0) + +int main(void) { + int dims[2] = {3, 3}; + tensor* raw = t_alloc(2, dims); + CHECK(raw != NULL); + for (int i = 0; i < 9; ++i) raw->storage->data[i] = (float)(i + 1) * 0.1f; + ag_tensor* input = ag_from_owned_tensor(raw, 1); + ag_tensor* transpose = ag_transpose(input, 0, 1); + ag_tensor* product = ag_matmul(input, transpose); + ag_tensor* rows = ag_sum(product, 1, 0); + ag_tensor* loss = ag_sum(rows, 0, 0); + ag_tensor* values[] = {input, transpose, product, rows, loss}; + CHECK(loss != NULL && ag_backward(loss) == 0); + tensor* single_step[5]; + for (int i = 0; i < 5; ++i) { + single_step[i] = t_clone(values[i]->grad); + CHECK(single_step[i] != NULL); + } + int failed_passes = 0; + for (int leaves = 0; leaves <= 1; ++leaves) { + int completed = 0, previous_failures = failed_passes; + ag_backward_options options = ag_backward_default_options(); + if (leaves) options.retention = AG_GRAD_RETAIN_LEAVES; + for (int budget = 0; budget < 512; ++budget) { + tensor* snapshots[5]; + for (int i = 0; i < 5; ++i) { + snapshots[i] = t_clone(values[i]->grad); + CHECK(snapshots[i] != NULL); + } + failures = 0; + remaining = budget; + int status = ag_backward_ex(loss, &options); + remaining = -1; + if (status != 0) { + ++failed_passes; + for (int i = 0; i < 5; ++i) { + CHECK(values[i]->grad != NULL && values[i]->graph_index == -1); + for (int j = 0; j < tensor_numel(snapshots[i]); ++j) + CHECK(values[i]->grad->storage->data[tensor_flat_index(values[i]->grad, j)] == + snapshots[i]->storage->data[j]); + } + } else { + for (int i = 0; i < 5; ++i) { + CHECK(values[i]->grad != NULL && values[i]->graph_index == -1); + for (int j = 0; j < tensor_numel(snapshots[i]); ++j) { + float contribution = !leaves || i == 0 + ? single_step[i]->storage->data[j] : 0.0f; + float expected = snapshots[i]->storage->data[j] + contribution; + CHECK(fabsf(values[i]->grad->storage->data[tensor_flat_index(values[i]->grad, j)] - + expected) < 1e-5f); + } + } + } + for (int i = 0; i < 5; ++i) t_free(snapshots[i]); + if (failures == 0) { completed = status == 0; break; } + } + CHECK(failed_passes > previous_failures && completed); + } + for (int i = 4; i >= 0; --i) { + t_free(single_step[i]); + ag_tensor_release(values[i]); + } + printf("allocation rollback: %d failing passes validated\n", failed_passes); + return 0; +} diff --git a/tests/unit/autograd/test_autograd_backward.c b/tests/unit/autograd/test_autograd_backward.c index b42094d..403beff 100644 --- a/tests/unit/autograd/test_autograd_backward.c +++ b/tests/unit/autograd/test_autograd_backward.c @@ -1,6 +1,7 @@ #include "./../../../include/tensorlib/autograd.h" #include "./../../../include/tensorlib/autograd_internal.h" #include "./../../fixtures/test_common.h" +#include "../../../src/tensor/tensor_alloc_internal.h" static ag_tensor* make_ag(int ndim, const int* dims, const float* values, int requires_grad) { tensor* raw = t_alloc(ndim, dims); @@ -36,6 +37,8 @@ TEST(test_backward_stats_are_disabled_by_default_and_resettable) { ag_tensor* square = ag_mul(input, input); ag_tensor* loss = ag_mean(square, 0, 0); ag_backward_stats stats; + tensor_alloc_stats_enable(1); + tensor_alloc_stats_reset(); ag_backward_stats_reset(); ag_backward_stats_enable(1); ASSERT_EQ_INT(ag_backward(loss), 0); @@ -43,11 +46,13 @@ TEST(test_backward_stats_are_disabled_by_default_and_resettable) { ag_backward_stats_read(&stats); ASSERT_EQ_INT((int)stats.operation_calls[AG_OP_MEAN], 1); ASSERT_EQ_INT((int)stats.operation_calls[AG_OP_MUL], 1); + ASSERT_TRUE(stats.operation_allocated_bytes[AG_OP_MUL] > 0); ASSERT_TRUE(stats.traversal_seconds >= 0.0); ASSERT_TRUE(stats.merge_seconds >= 0.0); ag_backward_stats_reset(); ag_backward_stats_read(&stats); ASSERT_EQ_INT((int)stats.operation_calls[AG_OP_MUL], 0); + tensor_alloc_stats_enable(0); ag_tensor_release(loss); ag_tensor_release(square); ag_tensor_release(input); } @@ -269,8 +274,50 @@ TEST(test_backward_rejects_modified_intermediate_output) { ag_tensor_release(input); } +static void retained_chain_peak(ag_grad_retention retention, size_t* result) { + int dims[1] = {4096}; + ag_backward_options options = {retention}; + tensor_alloc_stats stats; + tensor_alloc_stats_enable(1); + tensor_alloc_stats_reset(); + tensor* raw = t_alloc(1, dims); + tensor* seed = t_alloc(1, dims); + for (int i = 0; i < dims[0]; ++i) { + raw->storage->data[i] = 1.0f; + seed->storage->data[i] = 1.0f; + } + ag_tensor* input = ag_from_owned_tensor(raw, 1); + ag_tensor* output = input; + ag_tensor_retain(output); + for (int i = 0; i < 8; ++i) { + ag_tensor* next = ag_mul_scalar(output, 2.0f); + ASSERT_NOT_NULL(next); + ag_tensor_release(output); + output = next; + } + ASSERT_EQ_INT(ag_backward_with_grad_ex(output, seed, &options), 0); + ASSERT_EQ_FLOAT(input->grad->storage->data[0], 256.0f); + tensor_alloc_stats_read(&stats); + size_t peak = stats.peak_live_bytes; + ASSERT_EQ_INT(ag_backward_with_grad_ex(output, seed, &options), 0); + ASSERT_EQ_FLOAT(input->grad->storage->data[0], 512.0f); + ag_tensor_release(output); ag_tensor_release(input); t_free(seed); + tensor_alloc_stats_read(&stats); + ASSERT_EQ_INT(stats.live_bytes, 0); + tensor_alloc_stats_enable(0); + *result = peak; +} + +TEST(test_leaf_retention_bounds_live_intermediate_gradients) { + size_t all = 0, leaves = 0; + retained_chain_peak(AG_GRAD_RETAIN_ALL, &all); + retained_chain_peak(AG_GRAD_RETAIN_LEAVES, &leaves); + ASSERT_TRUE(leaves + 4 * 4096 * sizeof(float) < all); +} + int main(void) { printf("== autograd_backward.c ==\n"); + RUN_TEST(test_leaf_retention_bounds_live_intermediate_gradients); RUN_TEST(test_backward_chain_computes_weight_gradients); RUN_TEST(test_backward_stats_are_disabled_by_default_and_resettable); RUN_TEST(test_shared_dag_accumulates_each_branch_once); diff --git a/tests/unit/autograd/test_autograd_core.c b/tests/unit/autograd/test_autograd_core.c index 4d320dd..e8285e6 100644 --- a/tests/unit/autograd/test_autograd_core.c +++ b/tests/unit/autograd/test_autograd_core.c @@ -1,6 +1,7 @@ #include #include "./../../../include/tensorlib/autograd.h" #include "./../../fixtures/test_common.h" +#include "../../../src/tensor/tensor_alloc_internal.h" static int contexts_freed = 0; @@ -129,8 +130,25 @@ TEST(test_detach_stops_gradient_tracking_and_detects_alias_mutation) { ASSERT_NULL(ag_detach(NULL)); } +TEST(test_graph_profile_separates_nodes_from_tensor_storage) { + tensor_alloc_stats stats; + tensor_alloc_stats_enable(1); + tensor_alloc_stats_reset(); + ag_tensor* input = ag_from_owned_tensor(make_scalar(2.0f), 1); + ag_tensor* output = ag_mul(input, input); + ASSERT_NOT_NULL(output); + tensor_alloc_stats_read(&stats); + ASSERT_TRUE(stats.auxiliary_allocations[TENSOR_ALLOC_GRAPH] >= 5); + ASSERT_TRUE(stats.auxiliary_bytes[TENSOR_ALLOC_GRAPH] >= sizeof(ag_node)); + ag_tensor_release(output); ag_tensor_release(input); + tensor_alloc_stats_read(&stats); + ASSERT_EQ_INT(stats.live_bytes, 0); + tensor_alloc_stats_enable(0); +} + int main(void) { printf("== autograd_core.c ==\n"); + RUN_TEST(test_graph_profile_separates_nodes_from_tensor_storage); RUN_TEST(test_owned_tensor_initializes_normalized_leaf); RUN_TEST(test_owned_tensor_rejects_null); RUN_TEST(test_tensor_retain_keeps_value_alive); diff --git a/tests/unit/benchmark/test_bench_harness.c b/tests/unit/benchmark/test_bench_harness.c index 12d06f7..861eebc 100644 --- a/tests/unit/benchmark/test_bench_harness.c +++ b/tests/unit/benchmark/test_bench_harness.c @@ -23,6 +23,7 @@ int main(void) bench_profile quick = bench_profile_named("quick"); bench_profile full = bench_profile_named("full"); bench_profile smoke = bench_profile_named("smoke"); + bench_profile fixed = bench_profile_named("fixed"); bench_measurement result; int calls = 0; @@ -44,6 +45,13 @@ int main(void) fprintf(stderr, "smoke measurement produced invalid statistics\n"); return 1; } + calls = 0; + if (bench_measure(counted_operation, &calls, &fixed, &result) != 0 || + calls != 5 || result.checksum != 5.0 || + result.iterations_per_sample != 1) { + fprintf(stderr, "fixed profile must execute exactly five operations\n"); + return 1; + } printf("benchmark harness tests passed\n"); return 0; } diff --git a/tests/unit/tensor/test_tensor_alloc.c b/tests/unit/tensor/test_tensor_alloc.c index f8b3f43..c309a41 100644 --- a/tests/unit/tensor/test_tensor_alloc.c +++ b/tests/unit/tensor/test_tensor_alloc.c @@ -79,6 +79,32 @@ TEST(test_t_alloc_scalar_ndim_zero) { t_free(t); } +TEST(test_profile_counts_metadata_and_materialized_copies) { + int dims[2] = {2, 3}; + tensor_alloc_stats before, after; + tensor_alloc_stats_enable(1); + tensor_alloc_stats_reset(); + tensor* base = t_alloc(2, dims); + for (int i = 0; i < 6; ++i) base->storage->data[i] = (float)i; + tensor_alloc_stats_read(&before); + tensor* view = t_transpose(base, 0, 1); + tensor* copy = t_clone(view); + tensor_alloc_stats_read(&after); + ASSERT_TRUE(after.auxiliary_allocations[TENSOR_ALLOC_METADATA] > + before.auxiliary_allocations[TENSOR_ALLOC_METADATA]); + ASSERT_EQ_INT(after.copied_bytes, 6 * sizeof(float)); + tensor_alloc_stats_reset_counters(); + tensor_alloc_stats_read(&after); + ASSERT_EQ_INT(after.copied_bytes, 0); + ASSERT_EQ_INT(after.auxiliary_bytes[TENSOR_ALLOC_METADATA], 0); + ASSERT_EQ_INT(after.live_bytes, 12 * sizeof(float)); + t_free(copy); t_free(view); t_free(base); + tensor_alloc_stats_enable(0); + tensor_alloc_record_copy(99); + tensor_alloc_stats_read(&after); + ASSERT_EQ_INT(after.copied_bytes, 0); +} + TEST(test_t_alloc_rejects_negative_ndim) { tensor* t = t_alloc(-1, NULL); ASSERT_NULL(t); @@ -309,6 +335,7 @@ TEST(test_alloc_rejects_nonpositive_and_overflowing_dimensions) { ASSERT_NULL(t_alloc(2, overflow)); } int main(void) { + RUN_TEST(test_profile_counts_metadata_and_materialized_copies); printf("== tensor_alloc.c ==\n"); RUN_TEST(test_s_alloc_basic); RUN_TEST(test_s_alloc_ndim_zero_gives_size_one);