Repository navigation
Expand file tree
/
Copy pathDefaultEngine.BinaryOp.cs
More file actions
1840 lines (1699 loc) · 111 KB
/
Copy pathDefaultEngine.BinaryOp.cs
File metadata and controls
1840 lines (1699 loc) · 111 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
using System;
using System.Reflection.Emit;
using System.Runtime.CompilerServices;
using NumSharp.Backends.Iteration;
using NumSharp.Backends.Kernels;
using NumSharp.Utilities;
namespace NumSharp.Backends
{
/// <summary>
/// Binary operation dispatch using IL-generated kernels.
/// </summary>
public partial class DefaultEngine
{
// Call-invariant per-operand flag arrays for the NDIter routes
// (Wave 2.2): identical on every call, so allocating them per call
// was pure small-N overhead. The operand arrays stay per-call --
// the iterator stores the reference (_operands) and the overlap
// machinery can construct nested iterators (np.copyto) on the same
// thread, so thread-static reuse would alias live iterators.
private static readonly NDIterPerOpFlags[] s_binaryIterFlags =
{
NDIterPerOpFlags.READONLY | NDIterPerOpFlags.OVERLAP_ASSUME_ELEMENTWISE_PER_OP,
NDIterPerOpFlags.READONLY | NDIterPerOpFlags.OVERLAP_ASSUME_ELEMENTWISE_PER_OP,
NDIterPerOpFlags.WRITEONLY | NDIterPerOpFlags.OVERLAP_ASSUME_ELEMENTWISE_PER_OP,
};
// Element-count cutoff below which a SAME-DTYPE single-broadcast op skips
// the NDIter route in favour of the lighter direct SimdChunk kernel (see
// the gate in TryExecuteBinaryOpViaNDIter). NDIter's multi-operand
// construction costs a fixed ~0.5 µs that dominates a tiny broadcast:
// measured 1.8-2.3× faster on 1K row/col/3-D float64 broadcasts. The
// direct SimdChunk kernel matches or beats NDIter up to ~16K elements;
// above that NDIter's coalescing + vectorization pulls ahead, so the
// cutoff sits safely below the crossover (with margin for host-regime
// measurement noise). Env override NS_BROADCAST_DIRECT_MAX for tuning.
internal static readonly long DirectBroadcastMaxElements =
long.TryParse(Environment.GetEnvironmentVariable("NS_BROADCAST_DIRECT_MAX"), out var t) && t >= 0
? t : 8192;
/// <summary>
/// Execute a binary operation using IL-generated kernels.
/// Handles type promotion, broadcasting, and kernel dispatch.
/// </summary>
/// <param name="lhs">Left operand</param>
/// <param name="rhs">Right operand</param>
/// <param name="op">Operation to perform</param>
/// <param name="@out">Optional provided output (NumPy ufunc out=): the
/// result is written into it (dtype must be same_kind-castable from the
/// loop dtype, shape joins the broadcast without stretching) and the
/// same instance is returned.</param>
/// <param name="where">Optional bool write mask (NumPy ufunc where=):
/// only mask-true elements are computed/written; false slots keep the
/// prior out contents (uninitialized for a fresh result).</param>
/// <param name="dtype">Optional explicit loop dtype (NumPy ufunc dtype=):
/// overrides NEP50 promotion — the loop COMPUTES in this dtype (probed
/// 2.4.2: power(10,11,dtype=f64) = 1e11 exactly, no int wrap; the
/// f32-rounding of sqrt/power dtype=f32 lands even in a wider out).
/// Each input must be same_kind-castable to it, and a provided out is
/// validated against it rather than the promoted dtype.</param>
/// <param name="numpyPositions">True (every caller's default) to give np.fmax / np.fmin at a float32 /
/// float64 loop NumPy's position-dependent answer — the C library's ±0-tie / NaN-pair rule at the elements
/// NumPy runs outside its SIMD vectors (Default.FMaxFMin.Positions.cs, QA-20). False runs the kernels alone,
/// the vector rule at every position: a probe of what the kernels compute, never a NumPy answer.</param>
/// <returns>Result array with promoted type (or <paramref name="@out"/>)</returns>
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
internal unsafe NDArray ExecuteBinaryOp(NDArray lhs, NDArray rhs, BinaryOp op,
NDArray @out = null, NDArray where = null, NPTypeCode? dtype = null, bool numpyPositions = true)
{
// A string operand never reaches the numeric promotion table or a kernel: NumPy resolves the ufunc loop
// before anything else, and with a string operand it either refuses (its text) or computes on text (the next
// phase of the string support). Two typecode reads when no operand is a string.
// (An empty broadcast of a computing string loop is decided by the shapes: BinaryGuard returns that result.)
// NumPy's ufunc returns a 0-d result without out= as a SCALAR (PyArray_Return: np.add(np.array(1), 1) is
// np.int64(2)): every exit below routes a fresh result through it; out= comes back as given.
if (Strings.NDStringUfuncs.BinaryGuard(UfuncName(op), lhs, rhs, @out, dtype, where) is { } textResult)
return @out is null ? textResult.PyArray_Return() : textResult;
var lhsType = lhs.GetTypeCode;
var rhsType = rhs.GetTypeCode;
// Determine result type using NumPy type promotion rules
var resultType = np._FindCommonType(lhs, rhs);
// NumPy: true division (/) always returns float64 for integer types
// This matches Python 3 / NumPy 2.x semantics where / is "true division"
// Group 3 = float (Single, Double), Group 4 = Decimal
if (op == BinaryOp.Divide && resultType.GetGroup() < 3)
{
resultType = NPTypeCode.Double;
}
// NumPy Power promotion (NEP50) is exactly result_type(base, exp) — which
// _FindCommonType already computes, including the int-base/float-exp cases:
// - i32_arr ** i32_arr → int32 - f32_arr ** f64_arr → float64
// - i32_arr ** f32_arr → float64 - i16_arr ** f32_arr → float32
// - u16_arr ** f32_arr → float32 (uint16 fits in float32, probed 2.4.2)
// - f32_arr ** i32_arr → float64 (NEP50 strict)
// - 2 (weak int) ** f32_arr → float32 - 2 ** f64_arr → float64
// A prior override forced `lhsGroup<=2 && rhsGroup==3 → float64` for EVERY
// int-base ** float-exp, which wrongly upcast {bool,int8,int16,uint8,uint16,
// char} ** float32 to float64 (NumPy keeps float32) — a NumPy-1.x rule NEP50
// removed. Removed: result_type (above) is the single source of truth.
// ufunc dtype= (NumPy loop-signature override): the loop runs IN the
// requested dtype — inputs are cast (same_kind-validated, NumPy's
// UFuncTypeError text) and computation happens at that precision.
// Replaces the promoted dtype BEFORE the out/where branch so a
// provided out is validated against the dtype-overridden loop
// (probed: power(dtype=f32, out=i32) reports float32 → int32).
// NEP 50: a 0-d Python-int operand is packed into the final loop dtype with NumPy's setitem, which range-checks
// it (OverflowError "Python integer 1000 out of bounds for int8") where the kernels' cast used to wrap
// (int8 + 1000 gave [-23, -22]). NumPy packs while it resolves the loop: after it parsed out= (a read-only
// out) and where= (the mask's dtype), before the dtype= input casts — so those two checks run first here
// (ExecuteBinaryUfuncInto repeats them; they are idempotent).
if (@out is not null)
NumSharpException.ThrowIfNotWriteable(@out.Shape, "output array");
if (where is not null)
ValidateWhereMask(where);
CheckWeakPyIntOperand(lhs, rhs, dtype ?? resultType, loopFromPromotion: !dtype.HasValue);
if (dtype.HasValue && dtype.Value != resultType)
{
ValidateBinaryInputCasts(lhsType, rhsType, dtype.Value, UfuncName(op));
resultType = dtype.Value;
}
// Several NumPy ufuncs have NO bool loop — their smallest integer loop starts at int8,
// so a bool×bool input promotes to int8 (probed 2.4.2):
// left_shift/right_shift (bb->b absent), floor_divide (True//True -> 1:int8),
// remainder/mod (True%True -> 0:int8), power (True**True -> 1:int8),
// fmod (True fmod True -> 0:int8).
// Bump BEFORE the bool->logical remap below so these never take the bitwise-OR/AND path.
if (resultType == NPTypeCode.Boolean &&
(op == BinaryOp.LeftShift || op == BinaryOp.RightShift ||
op == BinaryOp.FloorDivide || op == BinaryOp.Mod || op == BinaryOp.Fmod || op == BinaryOp.Power))
resultType = NPTypeCode.SByte;
// NumPy bool arithmetic: the bool dtype has no integer add/multiply ufunc loop — `+`
// is logical OR and `*` is logical AND (so True + True == True, raw byte 1, not 2).
// The remap keys off the FINAL loop dtype (i.e. AFTER the dtype= override):
// add(bool, bool, dtype=i32) runs the i32 add loop and returns 2 (probed 2.4.2),
// while add(bool, bool) and add(bool, bool, dtype=bool) stay logical OR. The op
// remap makes every downstream kernel path (SIMD/scalar, same-type/mixed) emit
// the bitwise op and write a normalized 0/1 byte. (`-` has no bool loop and
// already throws like NumPy.)
if (resultType == NPTypeCode.Boolean)
{
if (op == BinaryOp.Add) op = BinaryOp.BitwiseOr;
else if (op == BinaryOp.Multiply) op = BinaryOp.BitwiseAnd;
// bool has no NaN domain: maximum/fmax == logical OR, minimum/fmin == logical AND
// (max(F,T)=T=OR, min(F,T)=F=AND). Remap so the bitwise kernels handle them.
else if (op == BinaryOp.Maximum || op == BinaryOp.FMax) op = BinaryOp.BitwiseOr;
else if (op == BinaryOp.Minimum || op == BinaryOp.FMin) op = BinaryOp.BitwiseAnd;
}
// NEP 50: a 0-d operand against an array is a weak Python scalar, and NumPy re-packs it into the final loop
// dtype with PyArray_Pack (npy_update_operand_for_scalar): an int goes through the correctly rounded double, so
// an int64 / uint64 past 2^53 rounds TWICE into a float32 loop (np.float32 array + (2**60 + 2**36 + 1) adds
// 2^60), where the kernels' operand cast rounds once (2^60 + 2^37). Such an operand is re-packed here, before
// either route; for every other operand the cast is the pack and nothing is allocated. The temp is disposed at
// every exit (no result aliases it: each route writes a fresh output or out=).
using var weakInt = AdoptWeakIntOperand(ref lhs, ref rhs, resultType);
if (weakInt is not null)
{
lhsType = lhs.GetTypeCode;
rhsType = rhs.GetTypeCode;
}
// np.fmax / np.fmin at a float32 / float64 loop: NumPy's answer to a ±0 tie or a NaN pair depends on whether
// the element falls inside a whole SIMD vector of its inner-loop call or outside one (QA-20). The C library's
// answers at NumPy's scalar positions are computed here, from the untouched inputs (an out= may alias one), the
// kernels then run as for every op, and those positions are rewritten (Default.FMaxFMin.Positions.cs). Here,
// after the loop dtype is final and the early validations ran, so the positions see exactly the operands the
// kernels will; a shape or out= cast error is still raised by the kernels' run (the positions skip such a call).
if (numpyPositions && (op == BinaryOp.FMax || op == BinaryOp.FMin)
&& (resultType == NPTypeCode.Single || resultType == NPTypeCode.Double))
{
var patches = NumPyFMaxFMinPatches(lhs, rhs, op == BinaryOp.FMax, resultType, @out, where);
var result = ExecuteBinaryOpResolved(lhs, rhs, op, lhsType, rhsType, resultType, @out, where);
// The positions are written into the result first; only then does a 0-d one become the scalar.
patches?.ApplyTo(result, this);
return @out is null ? result.PyArray_Return() : result;
}
var resolved = ExecuteBinaryOpResolved(lhs, rhs, op, lhsType, rhsType, resultType, @out, where);
return @out is null ? resolved.PyArray_Return() : resolved;
}
/// <summary>
/// The dispatch half of <see cref="ExecuteBinaryOp"/>: everything after the loop dtype is resolved and the early
/// validations ran — the out= / where= routes, the 0-d × 0-d shortcut, the trivial contiguous loop, the NDIter
/// route and the direct kernels. Split out so np.fmax / np.fmin can wrap it with their position step without
/// resolving the call twice.
/// </summary>
/// <param name="lhs">The first operand, after the weak-scalar adjustments (any temp it is stays alive in the
/// caller until this returns).</param>
/// <param name="rhs">The second operand, likewise.</param>
/// <param name="op">The op, after the bool remaps (bool add/multiply/max/min run as bitwise OR / AND).</param>
/// <param name="lhsType">The first operand's dtype.</param>
/// <param name="rhsType">The second operand's dtype.</param>
/// <param name="resultType">The final loop dtype.</param>
/// <param name="out">The out= array (writeability already checked), or null.</param>
/// <param name="where">The where= mask (already validated as bool), or null.</param>
/// <param name="kOrderResolved">True when the operands were already transposed into NumPy's output order by this
/// method's own K-order step (its recursive call): the step is not taken twice.</param>
/// <returns>The result: <paramref name="out"/> when given, else a fresh array, laid out in NumPy's ufunc output
/// order (the operands' K order; C for C-contiguous operands).</returns>
/// <exception cref="IncorrectShapeException">The operands do not broadcast, or out= does not have the broadcast
/// shape (NumPy's texts).</exception>
/// <exception cref="TypeError">out= cannot hold the loop dtype under same_kind casting.</exception>
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private unsafe NDArray ExecuteBinaryOpResolved(NDArray lhs, NDArray rhs, BinaryOp op, NPTypeCode lhsType,
NPTypeCode rhsType, NPTypeCode resultType, NDArray @out, NDArray where,
bool kOrderResolved = false)
{
// ufunc out=/where= path (Wave 2.1): the loop dtype above is final
// (NumPy resolves the loop from the INPUTS; out only constrains the
// final cast), so branch after promotion and before any allocation.
// NumPy likewise disables the trivial loop when a wheremask or
// provided out needs the full iterator (ufunc_object.c:2213).
if (@out is not null || where is not null)
{
// NumPy keeps the trivial loop for a provided out when every operand is trivially iterable
// (try_trivial_single_output_loop); only a mask, a cast or a layout / overlap the loop cannot take
// needs the iterator.
if (where is null && TryTrivialBinaryOpInto(lhs, rhs, op, lhsType, rhsType, resultType, @out))
return @out;
return ExecuteBinaryUfuncInto(lhs, rhs, op, lhsType, rhsType, resultType, @out, where);
}
// Handle scalar × scalar case
if (lhs.Shape.IsScalar && rhs.Shape.IsScalar)
{
return ExecuteScalarScalar(lhs, rhs, op, resultType);
}
// NumPy's ufunc output layout (parity QA-36): a fresh result follows the operands' K order, so a permuted or F
// operand keeps its memory order. The routes below write C (or F for an all-F set), so when the K layout is
// something else the operands are transposed into it (views: a dense permuted operand becomes C-contiguous) and
// the C result is relabelled. C-ordered operands (MayNeedKOrder) and a set NumPy's trivial loop lays out F,
// which the routes lay out F themselves (plain F strides, TrivialLoopLaysOut), skip it without a broadcast or a
// plan; an F set with a cast N-D input or a broadcast is NpyIter's, whose extent-1 strides only the plan gives.
if (!kOrderResolved && MayNeedKOrder(lhs.Shape, rhs.Shape)
&& !TrivialLoopLaysOut(lhs, rhs, lhsType != resultType, rhsType != resultType, out _))
{
var (kl, _) = Broadcast(lhs.Shape, rhs.Shape);
if (TryKOrderPlan(kl.dimensions, new[] { lhs.Shape, rhs.Shape }, out var kLayout, out var kPerm))
{
int knd = kl.NDim;
NDArray lt = IntoKOrder(lhs, knd, kPerm), rt = IntoKOrder(rhs, knd, kPerm);
try
{
var kr = ExecuteBinaryOpResolved(lt ?? lhs, rt ?? rhs, op, lhsType, rhsType, resultType, null, null,
kOrderResolved: true);
return NDArray.AdoptLayout(kr, kLayout);
}
finally
{
lt?.Dispose();
rt?.Dispose();
}
}
}
// NEP50 weak-scalar fast-path enablement. A Python-scalar literal arrives wrapped at
// its OWN dtype (`a * 2` -> int32 0-d; `a * 2.0` -> float64 0-d), so `f32[] * 2`
// reaches here as f32×int32. resultType already reflects NumPy's weak promotion (here
// f32), but the same-dtype gate inside TryTrivialContiguousBinaryOp / the SIMD-viability
// gate inside TryExecuteBinaryOpViaNDIter key off the RAW operand dtypes and bail
// (f32 != int32), routing to the heavy per-element-convert mixed path — measured ~2.25×
// slower than the SimdScalarRight scalar-broadcast loop it should take (100K f32:
// 0.080 ms vs 0.035 ms). When the ARRAY operand already IS resultType and only the
// 0-d/size-1 scalar differs, materialize that scalar at resultType (one element —
// exactly what NumPy does, casting the scalar to the loop dtype), so both operands share
// resultType and the same-dtype SIMD paths below light up. When the array ALSO differs
// (e.g. int32[] * 2.5 -> float64) we leave it: the array genuinely needs a cast, which is
// the mixed path's job. (scalar×scalar already returned above; value is identical either
// way — the mixed path converts the same scalar to resultType per element.)
// NEP50 scalar-cast temp. Cast(..., copy:true) mints a FRESH 0-d/size-1 array and we
// REASSIGN the operand parameter to it, so the temp is unreachable once the op returns
// — an undisposed intermediate (a pooled buffer + a finalizable graph) reclaimable only
// by a future GC + finalizer pass. It is captured here and disposed at every exit below.
// Deliberately NOT an [NDScoped] boundary: ExecuteBinaryOp is the library's hottest path
// (every +,-,*,/,%,… on arrays), so a method scope would impose Open/Track/Dispose on ALL
// binary ops merely to reclaim a temp that ONLY this narrow array-op-scalar-needing-cast
// sub-path ever mints — the targeted dispose costs nothing on the common (same-dtype /
// array-array) path. Result buffers are always fresh (a kernel writes a new output from
// the operands), never an alias of this scalar temp, so disposing it at return is safe.
// Gate: UndisposedIntermediateTests.BinaryScalarCastTemp_IsDisposed.
NDArray scalarCastTemp = null;
if (lhsType != rhsType)
{
bool lhsScalarLike = lhs.Shape.IsScalar || lhs.Shape.size == 1;
bool rhsScalarLike = rhs.Shape.IsScalar || rhs.Shape.size == 1;
if (rhsScalarLike && !lhsScalarLike && lhsType == resultType)
{
scalarCastTemp = rhs = Cast(rhs, resultType, copy: true);
rhsType = resultType;
}
else if (lhsScalarLike && !rhsScalarLike && rhsType == resultType)
{
scalarCastTemp = lhs = Cast(lhs, resultType, copy: true);
lhsType = resultType;
}
}
// -------- O(1) trivial-loop bypass -----------------------------
// NumSharp analogue of NumPy's check_for_trivial_loop +
// try_trivial_single_output_loop (ufunc_object.c), which handle a
// single strided inner loop "without using the (heavy) iterator."
// When both operands share ONE contiguous layout (both C, or both F),
// have identical shape (no broadcast) and the same dtype as the result
// (no cast), a single linear walk over all three buffers visits the
// same logical element — so we route straight to the existing DirectIL
// SimdFull whole-array kernel and skip the NDIter MultiNew/Initialize
// construction (measured ~600-2000 ns/call: 22-24% of a small
// contiguous op, <=3% once n>=64K). Returns null for anything that is
// not trivially contiguous (broadcast/mixed-C-F/strided/cast/scalar-
// broadcast/unsupported emit) → falls through to the NDIter route
// below with behaviour unchanged.
{
var trivial = TryTrivialContiguousBinaryOp(lhs, rhs, op, lhsType, rhsType, resultType);
if (trivial is not null) { scalarCastTemp?.Dispose(); return trivial; }
}
// -------- NDIter Tier 3B fast path (all binary ops) -----------
// Routes through the NDIter inner-loop kernel factory, which
// collapses coalesce + SIMD dispatch (contig, SimdScalarLeft,
// SimdScalarRight, scalar-strided) into a single emitted kernel
// driven by NDIter's multi-operand iterator.
//
// Same-dtype: full SIMD path (CanSimdAllOperands passes inside
// the factory). Measured 2.3-4.7× wins across 12 variations,
// at parity with NumPy 2.x.
//
// Mixed-dtype: scalar body emits per-operand EmitConvertTo
// before EmitScalarOperation, mirroring the direct path's
// EmitConvertTo + EmitScalarOperation sequence. Vector body
// is null (factory drops to scalar-strided). Equivalent perf
// for the mixed-dtype cases the direct path used to handle.
{
var routed = TryExecuteBinaryOpViaNDIter(lhs, rhs, op, lhsType, rhsType, resultType);
if (routed is not null) { scalarCastTemp?.Dispose(); return routed; }
}
// Broadcast shapes
var (leftShape, rightShape) = Broadcast(lhs.Shape, rhs.Shape);
var cleanShape = leftShape.Clean();
// NumPy-aligned layout preservation: when EVERY non-scalar operand is strictly
// F-contig, allocate the result in F-order up front and skip the post-kernel
// copy. Pre-L3-a this branch ran with C-allocated result + `result.copy('F')`
// at the end. Allocating F here saves the copy AND lets the L3-a coalesce
// collapse to 1-D SimdFull (≈15× speedup for the 1K×1K F-contig case).
//
// The stricter "all-F" rule is required for kernel correctness: kernels still
// walk the result buffer with linear `i*elemSize` indexing (C-order coords).
// If result is F-contig but any input operand is neither C nor F (e.g. negative
// strides, partial broadcast), the kernel writes positions that don't match
// the input's logical coords. The legacy `result.copy('F')` path stays for
// the looser "any F, no strict C" case via the post-kernel branch below.
//
// A strictly F operand can still BROADCAST: a (2,1,2) F array against a (2,3,2) F
// array is "strict F" by its own flags, yet its stride-0 middle axis blocks the
// F-order merge below, so the coalesced iteration keeps several dims and the linear
// C-order write over an F result scrambles the values (every SIMD dtype under the
// small-broadcast cutoff, and float16 add/subtract/multiply/divide at any size,
// which always take this route). So an F result is allocated here only when every
// non-scalar operand has the result's own dims; a broadcast all-F set computes in C
// order and the post-kernel branch relabels it F, NumPy's layout for that set.
//
// Such a broadcast all-F set is instead walked with its axes REVERSED: the kernel's C-coordinate order over
// the reversed dims is the F memory order of the original ones, so the linear write fills an F-allocated
// result correctly, every operand is read along its own contiguous (or stride-0) axis, and no C result is
// copied to F afterwards. Elementwise, so the visiting order changes no value; NumPy's iterator walks such a
// set in F order too (K order). This is the float16 add / subtract / multiply / divide route for every size
// (plan api-parameter-parity X4: a (30,40,20) F * (30,1,20) F product read 1,200-element strides along the
// C-order inner axis, then copied the result to F).
bool allStrictF = AreAllOperandsStrictFContig(lhs, rhs, cleanShape);
bool walkReversed = allStrictF && HasNonScalarBroadcastOperand(lhs, rhs, cleanShape);
Shape resultShape = allStrictF
? new Shape(cleanShape.dimensions, 'F', copy: false)
: cleanShape;
// Allocate result
var result = NewArray(resultType, resultShape, false);
// Empty broadcast result: no elements to compute. The kernels below assume
// >= 1 element and walk stride-0 broadcast dims as if non-empty, corrupting
// memory when a sibling dim is 0 (e.g. (3,1,1) op (1,0,2) -> (3,0,2)).
// (The NDIter fast path above returns early for the same reason.)
if (result.size == 0)
{
scalarCastTemp?.Dispose();
return result;
}
// L3-a: pre-coalesce adjacent dims with compatible strides for BOTH operands
// (and the result). This collapses F-contig N-D to 1-D contig, so the path
// classifier promotes from `General` (≈13× slower) to `SimdFull`. Broadcast
// and arbitrary strided cases survive unchanged because their cross-axis
// stride relationships don't satisfy the merge condition.
int origNdim = resultShape.NDim;
long* coalShape = stackalloc long[origNdim > 0 ? origNdim : 1];
long* coalLhsStr = stackalloc long[origNdim > 0 ? origNdim : 1];
long* coalRhsStr = stackalloc long[origNdim > 0 ? origNdim : 1];
long* coalResStr = stackalloc long[origNdim > 0 ? origNdim : 1];
for (int d = 0; d < origNdim; d++)
{
// A reversed walk reads axis (ndim - 1 - d) as the d-th: the F result's strides then descend, so the
// coalescing below sees a C-ordered result and merges the axes the F operands run along.
int s = walkReversed ? origNdim - 1 - d : d;
coalShape[d] = resultShape.dimensions[s];
coalLhsStr[d] = leftShape.strides[s];
coalRhsStr[d] = rightShape.strides[s];
coalResStr[d] = resultShape.strides[s];
}
int coalNdim = CoalesceTernaryDimensions(coalShape, coalLhsStr, coalRhsStr, coalResStr, origNdim);
// Classify execution path using coalesced strides
ExecutionPath path = ClassifyPath(coalLhsStr, coalRhsStr, coalShape, coalNdim, resultType);
// Get kernel key
var key = new MixedTypeKernelKey(lhsType, rhsType, resultType, op, path);
// Get or generate kernel
var kernel = DirectILKernelGenerator.GetMixedTypeKernel(key);
if (kernel != null)
{
// Execute IL kernel using coalesced shape/strides
ExecuteKernelCoalesced(kernel, lhs, rhs, result, leftShape, rightShape,
coalShape, coalLhsStr, coalRhsStr, coalNdim);
}
else
{
// Fallback to legacy implementation
FallbackBinaryOp(lhs, rhs, result, op, leftShape, rightShape);
}
// NumPy F-output preservation for the LOOSER case (at least one strict-F operand
// but not all): result is currently C-contig (correct kernel output). Copy to F
// to match NumPy. The strict-all-F case skipped this branch by allocating F up
// front (a broadcast all-F set too, through the reversed walk).
if (!allStrictF && ShouldProduceFContigOutput(lhs, rhs, result.Shape))
{
scalarCastTemp?.Dispose();
// copy('F') mints a fresh F-contig array; the C-contig `result` the kernel wrote
// is now dead — dispose it rather than drop it to the finalizer (mechanism-3 leak).
var fResult = result.copy('F');
result.Dispose();
return fResult;
}
scalarCastTemp?.Dispose();
return result;
}
/// <summary>
/// Try to execute a binary op via NDIter Tier 3B for any dtype
/// combination (same or mixed). Returns the result array on
/// success, or null if the route is not applicable (broadcast
/// result exceeds int.MaxValue, NDIter not built for long-
/// shape arithmetic; unsupported op/dtype emit).
///
/// Allocates the output as F-contig when both inputs are strictly
/// F (matches the pre-existing direct-path rule from L3-b) and
/// picks the NDIter order accordingly: NPY_FORTRANORDER for
/// strict-F-both, NPY_CORDER everywhere else. NPY_CORDER also
/// handles the reversed-stride case correctly because NDIter
/// normalizes negative inner strides during init.
///
/// Same-dtype path (<paramref name="lhsType"/> == <paramref name="rhsType"/>
/// == <paramref name="resultType"/>): scalar body is
/// <see cref="DirectILKernelGenerator.EmitScalarOperation"/>; vector
/// body is supplied when the dtype and op both support SIMD.
///
/// Mixed-dtype path: scalar body emits a load-shuffle that
/// converts each input from its source dtype to the result
/// dtype before invoking EmitScalarOperation — mirrors the
/// direct path's EmitConvertTo + EmitScalarOperation sequence
/// in EmitGeneralLoop / EmitChunkLoop. Vector body is null
/// because Tier 3B's <c>CanSimdAllOperands</c> rejects mixed
/// dtypes; factory drops straight to the scalar-strided loop.
///
/// After the kernel runs, applies the "looser-F" post-copy step
/// that the direct path uses: if the result is C-contig but the
/// NumPy-aligned rule says it should be F (at least one strict-F
/// input, no strict-C input), return <c>result.copy('F')</c>.
/// </summary>
private unsafe NDArray? TryExecuteBinaryOpViaNDIter(
NDArray lhs, NDArray rhs, BinaryOp op,
NPTypeCode lhsType, NPTypeCode rhsType, NPTypeCode resultType)
{
// Broadcast → clean shape so we know what the result looks like.
var (leftShape, rightShape) = Broadcast(lhs.Shape, rhs.Shape);
var cleanShape = leftShape.Clean();
// (Half,Half)→Half add/sub/mul/div DECLINE Tier 3B: the direct route serves
// them with the bit-exact SIMD widen-compute-narrow kernels
// (Binary.Arith.Half.cs — float32 compute + Giesen RTNE narrow, NumPy's
// exact HALF loop incl. the NaN-payload operand-order pin), while this
// route's scalar inner-loop body bridges Half through DOUBLE (its hardware
// (double)Half widen QUIETS sNaN before the op and double-rounds
// differently on exponent-gap sums). Contiguous same-shape pairs already
// took the trivial bypass; this redirect sends the scalar-broadcast
// layouts to SimdScalarLeft/Right and leaves strided on the same
// EmitScalarOperation numerics as before (via SimdChunk).
// Power joins them (plan api-parameter-parity X13): HALF_power is powf on the halves widened bit for bit, and
// the same kernels run it with a vector widen / narrow around a per-lane powf.
if (lhsType == NPTypeCode.Half && rhsType == NPTypeCode.Half && resultType == NPTypeCode.Half
&& (op == BinaryOp.Add || op == BinaryOp.Subtract || op == BinaryOp.Multiply || op == BinaryOp.Divide
|| op == BinaryOp.Power))
return null;
// NDIter's internal shape arithmetic is int-bounded; route only
// when the broadcast result fits. Pre-existing test
// LongIndexingBroadcastTest exercises the > int.MaxValue path via
// the direct allocator (which is also int-limited but doesn't
// throw on the shape calc itself). Falling through to the direct
// path keeps the prior behaviour for those edge cases.
if (cleanShape.size < 0) return null;
for (int i = 0; i < cleanShape.NDim; i++)
if (cleanShape.dimensions[i] > int.MaxValue) return null;
// Small SAME-DTYPE single-broadcast fast path: for a tiny result the
// NDIter multi-operand construction (~0.5 µs) dominates, and the direct
// SimdChunk kernel (reached by falling through to the direct route below)
// is measurably faster there — 1.8-2.3× on 1K row/col/3-D float64
// broadcasts, bringing them from ~0.55× NumPy to faster-than-NumPy.
// Returning null hands the op to the direct path in ExecuteBinaryOp.
//
// The route is gated to exactly the cases the direct SimdChunk kernel is
// proven BIT-IDENTICAL to NDIter on (verified by the oracle FuzzMatrix
// AND the layout-parity integration gates):
// • identical dtypes — the direct path's mixed-dtype convert loop
// diverges from NDIter on some NEP50 promotions;
// • a SIMD-capable op+dtype — Half/Decimal/Complex (and the scalar-only
// ops Mod/Power/FloorDivide/ATan2) take a scalar path whose specials
// diverge (e.g. float16 256 − inf);
// • EXACTLY ONE operand broadcast (XOR) — a genuine single-operand
// broadcast (row/col/N-D, one operand carries the full result shape).
// A kron-style DOUBLE broadcast (both stretched on complementary axes)
// leaves outputs unwritten in SimdChunk; a same-shape non-broadcast op
// is either contiguous (already handled by the trivial bypass) or a
// strided view SimdChunk mishandles — both must stay on NDIter;
// • both operands C- or F-contiguous — SimdChunk mishandles a strided /
// trailing-size-1 / negative-stride operand that NDIter absorbs
// (the (N,1)-strided-view and reshape-view integration gates), so
// only clean-layout inputs take this route;
// • NOT every non-scalar operand strictly F — NumPy lays such a set out F,
// and the direct route can only write it in C and copy it to F (its F
// allocation is unsafe once an operand broadcasts, see ExecuteBinaryOp),
// which measured 4-6× slower than this route's F allocation (16³ float64:
// 7.1 µs vs 1.7 µs; NumPy 2.7 µs).
// Above DirectBroadcastMaxElements NDIter's coalescing/vectorization wins,
// so the op stays here.
if (lhsType == rhsType && lhsType == resultType
&& cleanShape.size < DirectBroadcastMaxElements
&& (leftShape.IsBroadcasted ^ rightShape.IsBroadcasted)
&& IsContiguousCorF(lhs.Shape) && IsContiguousCorF(rhs.Shape)
&& !AreAllOperandsStrictFContig(lhs, rhs, cleanShape)
&& DirectILKernelGenerator.CanUseSimdBinary(op, resultType))
return null;
// Mirror the direct path: F-allocate output when every non-scalar
// operand is strict-F. Otherwise default to C and let the
// post-kernel "looser-F" copy step rectify when needed.
bool allStrictFContig = AreAllOperandsStrictFContig(lhs, rhs, cleanShape);
Shape resultShape = allStrictFContig
? new Shape(cleanShape.dimensions, 'F', copy: false)
: cleanShape;
var result = NewArray(resultType, resultShape, false);
// Empty broadcast result (a stride-0 broadcast dim alongside a zero-size
// dim, e.g. (3,1,1) op (1,0,2) -> (3,0,2)): there is nothing to compute.
// Returning here is REQUIRED — the NDIter element-wise path corrupts the
// heap when driven over a 0-element broadcast (the direct kernel path below
// is guarded the same way). Matches NumPy: empty op -> empty result.
if (result.size == 0)
return result;
// Order selection — see method-summary comment.
var order = allStrictFContig
? NPY_ORDER.NPY_FORTRANORDER
: NPY_ORDER.NPY_CORDER;
// SIMD viability: requires equal dtypes (CanSimdAllOperands in
// the Tier 3B factory enforces this anyway, but we short-circuit
// here to keep the vector body null when known to be unusable).
// Op gate: Decimal/Half/Complex go scalar-only (CanUseSimd
// returns false for them); Mod/Power/FloorDivide/ATan2 go
// scalar-only via CanUseSimdForOp.
bool sameDtype = lhsType == rhsType && lhsType == resultType;
bool simdViable = sameDtype
&& DirectILKernelGenerator.CanUseSimdBinary(op, resultType);
// NOTE (Wave 4, measured): the buffered-cast route (NumPy's ufunc
// config — cast inputs to the computation dtype in 8192-element
// windows, then run the same-dtype SIMD body) was implemented and
// A/B-measured here, and LOST to this fused per-element-convert
// path for every SIMD-able binary op class on i9-13900K/Release:
// add contig 2M: buffered 2.20 ms vs fused 1.49 ms
// add strided 1M: buffered 3.18 ms vs fused 2.98 ms
// div contig 2M: buffered 1.72 ms vs fused 1.61 ms
// The extra buffer round-trip (~16 B/element of L2 traffic + window
// machinery) outweighs the SIMD gain on cheap ops. NumPy buffers
// because its AOT C loops cannot fuse casts; our runtime IL CAN —
// the fused path also beats NumPy itself (i32+f64 4M: 5.8-6.9 ms vs
// NumPy 7.3-7.6 ms). Promoting UNARY math ops (sqrt/exp/...) DO use
// the buffered-cast route (see DefaultEngine.UnaryOp) where the
// SIMD body wins 1.36x+. Revisit only with new A/B evidence.
//
// Build per-element scalar emit body. For same-dtype we just call
// EmitScalarOperation directly. For mixed-dtype we wrap it with
// a per-operand convert pass (the direct path's
// EmitGeneralLoop / EmitChunkLoop does the same).
// NumPy's elementwise NaN operand order (float add/multiply keep the SECOND operand's NaN when both are
// NaN): see DirectILKernelGenerator.NaNOperandOrder.cs.
Action<ILGenerator> scalarBody;
if (sameDtype)
{
// The one definition every caller of this kernel key shares (the reduction's add forms too).
scalarBody = SameDtypeBinaryScalarBody(op, resultType);
}
else
{
NPTypeCode capLhs = lhsType, capRhs = rhsType, capRes = resultType;
BinaryOp capOp = op;
scalarBody = il => EmitMixedScalarBody(il, capLhs, capRhs, capRes, capOp);
}
Action<ILGenerator>? vectorBody = simdViable ? SameDtypeBinaryVectorBody(op, resultType) : null;
// Cache key MUST encode all three dtypes; mixed-dtype kernels
// are distinct from same-dtype ones for the same op. Packed key
// (no per-call string): npy_binop_{op}_{lhsType}_{rhsType}_{resultType}.
var cacheKey = InnerLoopKernelKey.Binary(op, lhsType, rhsType, resultType);
try
{
// COPY_IF_OVERLAP + OVERLAP_ASSUME_ELEMENTWISE mirrors NumPy's
// ufunc iterator flags (ufunc_object.c:1070): overlapping
// write/read operands force a write-back temporary; exact
// aliasing stays copy-free because this loop is elementwise.
// With a freshly allocated result this is a cheap extent check.
using var iter = NDIterRef.MultiNew(
3, new[] { lhs, rhs, result },
NDIterGlobalFlags.EXTERNAL_LOOP | NDIterGlobalFlags.COPY_IF_OVERLAP,
order,
NPY_CASTING.NPY_SAFE_CASTING,
s_binaryIterFlags);
// How NumPy would split this iteration into inner-loop calls — read only by the cold path of a
// float add/multiply whose broadcast operand is NaN (the struct outlives the call below).
BroadcastNaNRule nanRule = NumPyBroadcastNaNRule(op, lhsType, rhsType, resultType, resultType,
lhs.size, rhs.size, masked: false, iter.NDim, iter.HasNegPerm, iter.GetInnerStrideArray());
iter.ExecuteElementWiseBinary(lhsType, rhsType, resultType, scalarBody, vectorBody, cacheKey, &nanRule);
}
catch (NotSupportedException)
{
// EmitScalarOperation / EmitVectorOperation / EmitConvertTo
// can throw for combos they don't cover. Surface as null so
// the caller falls back to the direct path.
return null;
}
// Looser-F preservation: matches the post-kernel branch in the
// direct path. Triggers when the result is currently C-contig but
// the NumPy rule says it should be F because at least one input
// is strict-F and no input is strict-C.
if (!allStrictFContig && ShouldProduceFContigOutput(lhs, rhs, result.Shape))
{
var fResult = result.copy('F');
result.Dispose(); // C-contig kernel output now dead (mechanism-3 leak)
return fResult;
}
return result;
}
/// <summary>
/// O(1)-gated trivial-loop bypass — the NumSharp analogue of NumPy's
/// <c>check_for_trivial_loop</c> + <c>try_trivial_single_output_loop</c>
/// (ufunc_object.c), which handle a single strided inner loop "without
/// using the (heavy) iterator."
///
/// Fires only when the op needs neither broadcasting nor casting and
/// both operands share ONE contiguous layout, so element k of each
/// operand's buffer (from its own offset) is the same logical element:
/// • dtypes identical (lhs == rhs == result) — no cast;
/// • shapes identical (<see cref="Shape.Equals(Shape)"/>, which
/// compares size + dimensions and ignores strides) — no broadcast;
/// • neither operand broadcasted (stride-0 dim with extent > 1);
/// • both C-contiguous (→ C result) or both F-contiguous (→ F result).
/// 1-D arrays are both C and F; the C branch is tested first (C result
/// == F result there), so the F branch implies ndim > 1 strictly-F,
/// matching <see cref="AreAllOperandsStrictFContig"/>'s F-alloc rule.
/// Contiguous slices (offset != 0) qualify because
/// <see cref="ExecuteKernel"/> applies each operand's offset.
///
/// Routes to the SAME <see cref="ExecutionPath.SimdFull"/> DirectIL
/// kernel the post-NDIter fallback uses, so results are identical for
/// every dtype/op (the generator emits a SIMD or scalar loop per dtype).
/// Unsupported emits (e.g. bool subtract) throw inside the generator;
/// we catch and return null so the existing path raises/handles the
/// case exactly as before. Any non-trivial case returns null →
/// caller proceeds to the NDIter route.
/// </summary>
private unsafe NDArray? TryTrivialContiguousBinaryOp(
NDArray lhs, NDArray rhs, BinaryOp op,
NPTypeCode lhsType, NPTypeCode rhsType, NPTypeCode resultType)
{
// No cast: all three dtypes identical. Promotion cases (/ -> f64,
// int-base ** float -> f64, mixed dtypes) have resultType != input and
// are excluded here, deferring to the NDIter route that does the cast.
if (lhsType != rhsType || lhsType != resultType)
return null;
var ls = lhs.Shape;
var rs = rhs.Shape;
// Scalar-broadcast: exactly one operand is scalar/size-1 and the other
// is a contiguous array (size > 1). NumPy's trivial loop broadcasts 0-D
// operands this way. The scalar is read once, the array walked linearly,
// and the result takes the array operand's shape+layout. (Both size-1, or
// both size > 1, fall through to the equal-shape branch below.)
bool lhsScalarLike = ls.IsScalar || ls.size == 1;
bool rhsScalarLike = rs.IsScalar || rs.size == 1;
if (lhsScalarLike ^ rhsScalarLike)
{
// The result takes the array's shape only when the one-element operand adds no dimension: a (1,1)
// operand against a (7,) array broadcasts to (1,7) in NumPy, which this shortcut would return as (7,)
// (plan api-parameter-parity X19). Such a call takes the broadcasting routes below.
if (rhsScalarLike ? rs.NDim > ls.NDim : ls.NDim > rs.NDim)
return null;
return TryScalarBroadcastBinaryOp(
lhs, rhs, op, resultType,
array: rhsScalarLike ? lhs : rhs,
scalarIsRhs: rhsScalarLike);
}
// Identical logical shape (no broadcast). Shape.Equals compares size +
// dimensions and ignores strides/offset — exactly the "same shape,
// either layout" test we want.
if (!ls.Equals(rs))
return null;
// A stride-0 dim with extent > 1 breaks the linear-walk assumption even
// if the contiguity flags happen to look set; exclude explicitly.
if (ls.IsBroadcasted || rs.IsBroadcasted)
return null;
// One shared contiguous layout (C checked first; see summary).
bool bothC = ls.IsContiguous && rs.IsContiguous;
bool bothF = !bothC && ls.IsFContiguous && rs.IsFContiguous;
if (!bothC && !bothF)
return null;
// SimdFull kernel: ignores strides, walks result.size linearly. Emit may
// be unsupported for some op/dtype (e.g. bool '-') — fall through to the
// existing path so it raises (or handles) the case identically.
var key = new MixedTypeKernelKey(lhsType, rhsType, resultType, op, ExecutionPath.SimdFull);
MixedTypeKernel kernel;
try
{
kernel = DirectILKernelGenerator.GetMixedTypeKernel(key);
}
catch (NotSupportedException)
{
return null;
}
if (kernel == null)
return null;
// Reuse a canonical input shape (offset 0, owns its buffer, right order)
// instead of cloning dims + reconstructing strides/flags. bothF implies
// strictly column-major ls (1-D would have satisfied bothC first).
Shape resultShape = CanonicalResultShape(ls, bothF);
var result = NewArray(resultType, resultShape, false);
// Empty result: nothing to compute (the kernel assumes >= 1 element).
if (result.size == 0)
return result;
ExecuteKernel(kernel, lhs, rhs, result, ls, rs);
return result;
}
/// <summary>
/// Scalar-broadcast arm of the trivial-loop bypass: <c>array op scalar</c>
/// or <c>scalar op array</c> where the array operand is contiguous. Routes
/// to the existing <see cref="ExecutionPath.SimdScalarRight"/> /
/// <see cref="ExecutionPath.SimdScalarLeft"/> DirectIL kernel (scalar read
/// once, array walked linearly), skipping NDIter construction. The result
/// takes the array operand's shape and layout (C, or strictly-F) so the
/// linear write aligns with the linear array read. Returns null (→ NDIter)
/// when the array operand is non-contiguous or the emit is unsupported.
///
/// Callers guarantee identical dtypes (same-dtype gate in
/// <see cref="TryTrivialContiguousBinaryOp"/>), so the kernel key uses
/// <paramref name="resultType"/> for all three operand slots.
/// </summary>
private unsafe NDArray? TryScalarBroadcastBinaryOp(
NDArray lhs, NDArray rhs, BinaryOp op, NPTypeCode resultType,
NDArray array, bool scalarIsRhs)
{
var arrShape = array.Shape;
if (arrShape.IsBroadcasted)
return null;
bool isC = arrShape.IsContiguous;
bool isF = !isC && arrShape.IsFContiguous;
if (!isC && !isF)
return null; // strided/transposed array operand → NDIter
var path = scalarIsRhs ? ExecutionPath.SimdScalarRight : ExecutionPath.SimdScalarLeft;
var key = new MixedTypeKernelKey(resultType, resultType, resultType, op, path);
MixedTypeKernel kernel;
try
{
kernel = DirectILKernelGenerator.GetMixedTypeKernel(key);
}
catch (NotSupportedException)
{
return null;
}
if (kernel == null)
return null;
Shape resultShape = CanonicalResultShape(arrShape, isF);
var result = NewArray(resultType, resultShape, false);
if (result.size == 0)
return result;
// lhs/rhs keep their original operand positions; the SimdScalarRight/Left
// kernel reads the scalar side once and walks the array side linearly.
ExecuteKernel(kernel, lhs, rhs, result, lhs.Shape, rhs.Shape);
return result;
}
/// <summary>
/// The trivial loop for a binary ufunc WITH a provided output: NumPy's ufunc runs
/// <c>try_trivial_single_output_loop</c> for <c>out=</c> too whenever every operand is trivially iterable,
/// and so does this — the same whole-array DirectIL kernel the fresh-result trivial loop
/// (<see cref="TryTrivialContiguousBinaryOp"/>) calls, writing straight into <paramref name="out"/>, with no
/// iterator. The iterator route stays for everything else (a cast, a broadcast array, a strided or mixed
/// layout, an overlap to resolve).
/// </summary>
/// <param name="lhs">The first input.</param>
/// <param name="rhs">The second input.</param>
/// <param name="op">The binary op, after the bool remaps of <see cref="ExecuteBinaryOp"/>.</param>
/// <param name="lhsType">The first input's dtype.</param>
/// <param name="rhsType">The second input's dtype.</param>
/// <param name="resultType">The loop dtype.</param>
/// <param name="out">The provided output; written in place when the call is trivial.</param>
/// <returns>True when the result was written into <paramref name="out"/>; false (nothing written, nothing
/// raised) when the call is not trivially iterable or the kernel is not available, so the caller takes the
/// iterator route — which also raises every error (a read-only out, a bad cast, a shape mismatch).</returns>
/// <remarks>
/// <para>
/// Trivial means: one dtype for both inputs and the output (no cast); a writeable, non-broadcast output,
/// C- or F-contiguous; every array input of the output's exact dimensions, contiguous in the output's
/// order; at most one size-1 input, of rank no higher than the output's (it broadcasts, as a stride-0
/// scalar, without growing the shape — NumPy's 0-d operands); and no memory overlap that NumPy's
/// <c>COPY_IF_OVERLAP</c> would resolve: an array input is either disjoint from the output or the output
/// itself (same first element and, with the same dimensions and order, the same strides — the in-place
/// <c>np.add(x, 1, out=x)</c>), and a size-1 input is disjoint (it is read once, before the loop overwrites
/// anything). The overlaps NumPy's <c>PyArray_EQUIVALENTLY_ITERABLE_OVERLAP_OK</c> also accepts (an input
/// reading ahead of the output) go to the iterator, whose copy gives the same values.
/// </para>
/// <para>
/// Bit-identical to the iterator route: both run the same scalar/vector op bodies, and a float add /
/// multiply with a NaN broadcast operand applies NumPy's AVX2 scalar-broadcast region rule over the one
/// inner-loop call NumPy's trivial loop makes (<see cref="NumPyBroadcastNaNRule"/> computes the same split
/// for the iterator). What it saves is the iterator's construction — and, for float16, the per-element
/// scalar body: the iterator route has no float16 vector body (there is no <c>Vector256<Half></c>
/// arithmetic), while the DirectIL kernels serve float16 add / subtract / multiply / divide with the SIMD
/// widen-compute-narrow kernels (<c>DirectILKernelGenerator.TryGetHalfArithKernel</c>): 3.9 → ~0.5 ns per
/// element for <c>np.add(0-d, x, out=o)</c> over 1M float16 elements.
/// </para>
/// </remarks>
private unsafe bool TryTrivialBinaryOpInto(
NDArray lhs, NDArray rhs, BinaryOp op,
NPTypeCode lhsType, NPTypeCode rhsType, NPTypeCode resultType, NDArray @out)
{
if (DisableTrivialBinaryOut)
return false;
if (lhsType != rhsType || lhsType != resultType || @out.GetTypeCode != resultType)
return false;
// The shifts have their own kernels (NumPy's overflow rule); the DirectIL binary kernels are not theirs.
if (op == BinaryOp.LeftShift || op == BinaryOp.RightShift)
return false;
var os = @out.Shape;
// A read-only or broadcast output, or an empty one, is the iterator route's to raise / return.
if (!os.IsWriteable || os.IsBroadcasted || os.size == 0)
return false;
bool outC = os.IsContiguous;
bool outF = !outC && os.IsFContiguous;
if (!outC && !outF)
return false;
var ls = lhs.Shape;
var rs = rhs.Shape;
bool lhsScalar = ls.size == 1 && ls.NDim <= os.NDim;
bool rhsScalar = rs.size == 1 && rs.NDim <= os.NDim;
bool lhsArray = !lhsScalar && IsTrivialArrayOperand(ls, os, outC);
bool rhsArray = !rhsScalar && IsTrivialArrayOperand(rs, os, outC);
ExecutionPath path;
if (lhsArray && rhsArray)
path = ExecutionPath.SimdFull;
else if (lhsScalar && rhsArray)
path = ExecutionPath.SimdScalarLeft;
else if (lhsArray && rhsScalar)
path = ExecutionPath.SimdScalarRight;
else if (resultType == NPTypeCode.Half && IsHalfArithOp(op))
return TryHalfArithBroadcastInto(lhs, rhs, op, @out, outC);
else
return false; // two size-1 inputs filling a larger out, a broadcast array, a strided input: iterator
// Overlap: an array input must be disjoint from out or be out itself; a size-1 input must be disjoint.
if (!TrivialOverlapOk(lhs, @out, lhsArray) || !TrivialOverlapOk(rhs, @out, rhsArray))
return false;
MixedTypeKernel kernel;
try
{
kernel = DirectILKernelGenerator.GetMixedTypeKernel(
new MixedTypeKernelKey(resultType, resultType, resultType, op, path));
}
catch (NotSupportedException)
{
return false; // no DirectIL kernel for this (op, dtype): the iterator route handles / raises it
}
if (kernel == null)
return false;
int itemSize = resultType.SizeOf();
byte* lhsAddr = (byte*)lhs.Address + ls.offset * itemSize;
byte* rhsAddr = (byte*)rhs.Address + rs.offset * itemSize;
byte* outAddr = (byte*)@out.Address + os.offset * itemSize;
fixed (long* lhsStrides = ls.strides)
fixed (long* rhsStrides = rs.strides)
fixed (long* dims = os.dimensions)
{
kernel(lhsAddr, rhsAddr, outAddr, lhsStrides, rhsStrides, dims, os.NDim, os.size);
}
TrivialBinaryOutRuns++;
return true;
}
/// <summary>Whether <paramref name="op"/> is one of the float16 ufuncs the SIMD widen-compute-narrow kernels
/// serve (<c>DirectILKernelGenerator.TryGetHalfArithKernel</c>): add, subtract, multiply, divide and power (powf per
/// lane between the vector widen and narrow, plan api-parameter-parity X13).</summary>
/// <param name="op">The binary op.</param>
/// <returns>True for add, subtract, multiply, divide and power.</returns>
private static bool IsHalfArithOp(BinaryOp op)
=> op == BinaryOp.Add || op == BinaryOp.Subtract || op == BinaryOp.Multiply || op == BinaryOp.Divide
|| op == BinaryOp.Power;
/// <summary>
/// float16 add / subtract / multiply / divide into a C- or F-contiguous <paramref name="out"/> whose inputs
/// broadcast to it or are strided: the direct route's strided widen-compute-narrow kernel
/// (<c>HalfArithStrided</c>) writes straight into <paramref name="out"/>, walked in its memory order. NumPy
/// runs its iterator there, whose HALF loop is scalar; so did NumSharp, with a scalar float16 body of ~5 ns an
/// element (plan api-parameter-parity X15: numpy.ma's anom subtracts a column mean into an <c>out=</c>
/// buffer).
/// </summary>
/// <param name="lhs">The first float16 input.</param>
/// <param name="rhs">The second float16 input.</param>
/// <param name="op">Add, Subtract, Multiply or Divide.</param>
/// <param name="out">The float16 output: writeable, not broadcast, non-empty, C- or F-contiguous (checked by the
/// caller).</param>
/// <param name="outC">True when <paramref name="out"/> is C-contiguous, false when it is strictly F-contiguous
/// (then the axes are walked reversed, so the linear write follows its memory order).</param>
/// <returns>True when the result was written; false (nothing written) when an input does not broadcast to
/// <paramref name="out"/>'s dimensions or may overlap it — the iterator route raises NumPy's broadcast error
/// or resolves the overlap with its copy.</returns>
/// <remarks>
/// Bit-identical to the iterator route: NumPy's float16 arithmetic is elementwise with a per-element NaN
/// priority (no call-schedule rule), and the SIMD kernels are its HALF loop. An input may be <paramref name="out"/>
/// itself (the same first element, dimensions and strides: each element is read before it is written at the
/// same position); any other overlap goes to the iterator, as in <see cref="TrivialOverlapOk"/>.
/// </remarks>
private unsafe bool TryHalfArithBroadcastInto(NDArray lhs, NDArray rhs, BinaryOp op, NDArray @out, bool outC)
{
var os = @out.Shape;
int nd = os.NDim;
long* lStr = stackalloc long[nd > 0 ? nd : 1];
long* rStr = stackalloc long[nd > 0 ? nd : 1];
if (!HalfBroadcastStridesTo(lhs.Shape, os, lStr) || !HalfBroadcastStridesTo(rhs.Shape, os, rStr))
return false;