diff --git a/.gitignore b/.gitignore index ba76895ea..b80e4b899 100644 --- a/.gitignore +++ b/.gitignore @@ -284,6 +284,8 @@ docs/gates/logs/* !docs/gates/logs/20260907-issue41-primitive-cleanup/** !docs/gates/logs/20260907-issue42-scalar-pyc/ !docs/gates/logs/20260907-issue42-scalar-pyc/** +!docs/gates/logs/20260916-alu-bitfield-decomposition/ +!docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json !docs/gates/logs/20260915-issue126-snapshot-worklist/ !docs/gates/logs/20260915-issue126-snapshot-worklist/** !docs/gates/logs/20260915-issue126-field-order-canonical/ diff --git a/compiler/acir/include/acir/Dialect/ACIR/ACIROps.td b/compiler/acir/include/acir/Dialect/ACIR/ACIROps.td index 8cca000a6..1ee359e46 100644 --- a/compiler/acir/include/acir/Dialect/ACIR/ACIROps.td +++ b/compiler/acir/include/acir/Dialect/ACIR/ACIROps.td @@ -174,9 +174,27 @@ def ACIR_VarMulOp : ACIR_VarBinaryOp<"var.mul">; def ACIR_VarUDivOp : ACIR_VarBinaryOp<"var.udiv"> { let hasCanonicalizer = 1; } +def ACIR_VarSDivOp : ACIR_VarBinaryOp<"var.sdiv">; def ACIR_VarURemOp : ACIR_VarBinaryOp<"var.urem"> { let hasCanonicalizer = 1; } +def ACIR_VarSRemOp : ACIR_VarBinaryOp<"var.srem">; + +// One physical divider produces quotient and remainder together. The +// signed/word controls select PTO architectural semantics while both results +// remain tied to one shared operation through QueueGraph and PYC lowering. +def ACIR_VarDivRemOp : ACIR_Op<"var.divrem", [Pure]> { + let arguments = (ins ACIR_VarType:$lhs, ACIR_VarType:$rhs, + ACIR_VarType:$signed_mode, ACIR_VarType:$word_mode); + let results = (outs ACIR_VarType:$quotient, ACIR_VarType:$remainder); + let hasVerifier = 1; + let assemblyFormat = [{ + $lhs `,` $rhs `,` $signed_mode `,` $word_mode attr-dict `:` + qualified(type($lhs)) `,` qualified(type($rhs)) `,` + qualified(type($signed_mode)) `,` qualified(type($word_mode)) `->` + qualified(type($quotient)) `,` qualified(type($remainder)) + }]; +} def ACIR_VarAndOp : ACIR_VarBinaryOp<"var.and">; def ACIR_VarOrOp : ACIR_VarBinaryOp<"var.or">; def ACIR_VarXorOp : ACIR_VarBinaryOp<"var.xor">; @@ -356,6 +374,134 @@ def ACIR_VarInsertOp : ACIR_Op<"var.insert", [Pure]> { }]; } +// Architectural scalar ALU primitives. These remain first-class ACIR +// operations so Python designs do not encode word truncation, wrapping +// bitfield rotation, or signedness in expression trees. +class ACIR_VarAluBinaryOp : ACIR_Op + ]> { + let arguments = (ins ACIR_VarType:$lhs, ACIR_VarType:$rhs); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $lhs `,` $rhs attr-dict `:` qualified(type($lhs)) `->` + qualified(type($result)) + }]; +} + +def ACIR_VarAddwOp : ACIR_VarAluBinaryOp<"var.addw">; +def ACIR_VarSubwOp : ACIR_VarAluBinaryOp<"var.subw">; +def ACIR_VarAndwOp : ACIR_VarAluBinaryOp<"var.andw">; +def ACIR_VarOrwOp : ACIR_VarAluBinaryOp<"var.orw">; +def ACIR_VarXorwOp : ACIR_VarAluBinaryOp<"var.xorw">; +def ACIR_VarSllOp : ACIR_VarAluBinaryOp<"var.sll">; +def ACIR_VarSrlOp : ACIR_VarAluBinaryOp<"var.srl">; +def ACIR_VarSraOp : ACIR_VarAluBinaryOp<"var.sra">; +def ACIR_VarSllwOp : ACIR_VarAluBinaryOp<"var.sllw">; +def ACIR_VarSrlwOp : ACIR_VarAluBinaryOp<"var.srlw">; +def ACIR_VarSrawOp : ACIR_VarAluBinaryOp<"var.sraw">; +def ACIR_VarSminOp : ACIR_VarAluBinaryOp<"var.smin">; +def ACIR_VarUminOp : ACIR_VarAluBinaryOp<"var.umin">; +def ACIR_VarSmaxOp : ACIR_VarAluBinaryOp<"var.smax">; +def ACIR_VarUmaxOp : ACIR_VarAluBinaryOp<"var.umax">; +def ACIR_VarMulwOp : ACIR_VarAluBinaryOp<"var.mulw">; + +class ACIR_VarAluTernaryOp : ACIR_Op + ]> { + let arguments = (ins ACIR_VarType:$lhs, ACIR_VarType:$rhs, + ACIR_VarType:$aux); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $lhs `,` $rhs `,` $aux attr-dict `:` qualified(type($lhs)) `->` + qualified(type($result)) + }]; +} + +def ACIR_VarMaddOp : ACIR_VarAluTernaryOp<"var.madd">; +def ACIR_VarMaddwOp : ACIR_VarAluTernaryOp<"var.maddw">; +def ACIR_VarMsubOp : ACIR_VarAluTernaryOp<"var.msub">; + +class ACIR_VarBitfieldOp : ACIR_Op { + let arguments = (ins ACIR_VarType:$value, ACIR_VarType:$width, + ACIR_VarType:$offset); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $value `,` $width `,` $offset attr-dict `:` + qualified(type($value)) `,` qualified(type($width)) `,` + qualified(type($offset)) `->` qualified(type($result)) + }]; +} + +def ACIR_VarBitfieldExtractOp : ACIR_Op<"var.bitfield_extract", [Pure]> { + let arguments = (ins ACIR_VarType:$value, ACIR_VarType:$width, + ACIR_VarType:$offset, BoolAttr:$signed_mode); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $value `,` $width `,` $offset `signed_mode` $signed_mode attr-dict `:` + qualified(type($value)) `,` qualified(type($width)) `,` + qualified(type($offset)) `->` qualified(type($result)) + }]; +} + +def ACIR_VarBitfieldPopcountOp : ACIR_VarBitfieldOp<"var.bitfield_popcount">; +def ACIR_VarBitfieldClzOp : ACIR_VarBitfieldOp<"var.bitfield_clz">; +def ACIR_VarBitfieldCtzOp : ACIR_VarBitfieldOp<"var.bitfield_ctz">; +def ACIR_VarBitfieldClearOp : ACIR_VarBitfieldOp<"var.bitfield_clear">; +def ACIR_VarBitfieldSetOp : ACIR_VarBitfieldOp<"var.bitfield_set">; +def ACIR_VarBitfieldReverseBytesOp : ACIR_VarBitfieldOp<"var.bitfield_reverse_bytes">; + +def ACIR_VarBitfieldInsertOp : ACIR_Op<"var.bitfield_insert", [Pure]> { + let arguments = (ins ACIR_VarType:$value, ACIR_VarType:$source, + ACIR_VarType:$width, ACIR_VarType:$offset); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $value `,` $source `,` $width `,` $offset attr-dict `:` + qualified(type($value)) `,` qualified(type($source)) `,` + qualified(type($width)) `,` qualified(type($offset)) `->` + qualified(type($result)) + }]; +} + +def ACIR_VarSextLowOp : ACIR_Op<"var.sext_low", [Pure]> { + let arguments = (ins ACIR_VarType:$value, ACIR_VarType:$width); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $value `,` $width attr-dict `:` qualified(type($value)) `,` + qualified(type($width)) `->` + qualified(type($result)) + }]; +} + +def ACIR_VarZextLowOp : ACIR_Op<"var.zext_low", [Pure]> { + let arguments = (ins ACIR_VarType:$value, ACIR_VarType:$width); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $value `,` $width attr-dict `:` qualified(type($value)) `,` + qualified(type($width)) `->` + qualified(type($result)) + }]; +} + +def ACIR_VarCselOp : ACIR_Op<"var.csel", [Pure]> { + let arguments = (ins ACIR_VarType:$predicate, ACIR_VarType:$lhs, + ACIR_VarType:$rhs, ACIR_VarType:$negate_false); + let results = (outs ACIR_VarType:$result); + let hasVerifier = 1; + let assemblyFormat = [{ + $predicate `,` $lhs `,` $rhs `,` $negate_false attr-dict `:` + qualified(type($predicate)) `,` qualified(type($lhs)) `,` + qualified(type($rhs)) `,` qualified(type($negate_false)) `->` + qualified(type($result)) + }]; +} + def ACIR_VarGetOp : ACIR_Op<"var.get", [Pure]> { let arguments = (ins ACIR_VarType:$record, StrAttr:$field); let results = (outs ACIR_VarType:$result); diff --git a/compiler/acir/lib/Analysis/VariableAnalysis.cpp b/compiler/acir/lib/Analysis/VariableAnalysis.cpp index 8741cfe91..2655b845b 100644 --- a/compiler/acir/lib/Analysis/VariableAnalysis.cpp +++ b/compiler/acir/lib/Analysis/VariableAnalysis.cpp @@ -161,7 +161,10 @@ class StructuralValueInterner { for (Value operand : operation->getOperands()) operands.push_back(memo.lookup(operand)); const bool commutative = - isa(operation) || + isa( + operation) || (isa(operation) && (cast(operation).getPredicate() == "eq" || cast(operation).getPredicate() == "ne")); @@ -175,8 +178,7 @@ class StructuralValueInterner { << cast(frame.value).getResultNumber() << '('; llvm::interleaveComma(operands, stream); stream << ')'; - auto position = - interned.try_emplace(key, memo.lookup(frame.value)).first; + auto position = interned.try_emplace(key, memo.lookup(frame.value)).first; memo[frame.value] = position->getValue(); } return memo.lookup(value); @@ -275,6 +277,28 @@ ValueConstraint evaluateBinary(const ValueConstraint &left, return ValueConstraint::finiteSet(result); } +template +ValueConstraint evaluateTernary(const ValueConstraint &first, + const ValueConstraint &second, + const ValueConstraint &third, Fn &&evaluate) { + auto firstValues = exactValues(first); + auto secondValues = exactValues(second); + auto thirdValues = exactValues(third); + if (!firstValues || !secondValues || !thirdValues || + firstValues->size() * secondValues->size() * thirdValues->size() > 64) + return ValueConstraint::unknown(); + SmallVector result; + llvm::DenseSet unique; + for (uint64_t a : *firstValues) + for (uint64_t b : *secondValues) + for (uint64_t c : *thirdValues) { + uint64_t value = evaluate(a, b, c); + if (unique.insert(value).second) + result.push_back(value); + } + return ValueConstraint::finiteSet(result); +} + ValueConstraint operandConstraint(ArrayRef operands, unsigned index) { @@ -383,10 +407,8 @@ inferConstraint(Operation *operation, unsigned resultIndex, uint64_t lower = 0; uint64_t upper = 0; if (kind == "add") { - if (right->first > - std::numeric_limits::max() - left->first || - right->second > - std::numeric_limits::max() - left->second) + if (right->first > std::numeric_limits::max() - left->first || + right->second > std::numeric_limits::max() - left->second) return defaultConstraint(resultType); lower = left->first + right->first; upper = left->second + right->second; @@ -397,11 +419,9 @@ inferConstraint(Operation *operation, unsigned resultIndex, upper = left->second - right->first; } else { if ((left->first != 0 && - right->first > - std::numeric_limits::max() / left->first) || + right->first > std::numeric_limits::max() / left->first) || (left->second != 0 && - right->second > - std::numeric_limits::max() / left->second)) + right->second > std::numeric_limits::max() / left->second)) return defaultConstraint(resultType); lower = left->first * right->first; upper = left->second * right->second; @@ -409,12 +429,74 @@ inferConstraint(Operation *operation, unsigned resultIndex, return upper <= mask ? ValueConstraint::closedInterval(lower, upper) : defaultConstraint(resultType); }; + auto sext32 = [](uint64_t value) -> uint64_t { + uint64_t low = value & 0xffffffffULL; + return (low & 0x80000000ULL) ? low | 0xffffffff00000000ULL : low; + }; + auto aluBinary = [&](auto &&fn) { + ValueConstraint result = binary(std::forward(fn)); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + }; + auto aluWordBinary = [&](auto &&fn) { + return aluBinary([&](uint64_t lhs, uint64_t rhs) { + return sext32(fn(lhs & 0xffffffffULL, rhs & 0xffffffffULL)); + }); + }; + auto aluShift = [&](bool arithmetic, bool word, bool left) { + return aluBinary([&](uint64_t lhs, uint64_t rhs) { + unsigned amount = static_cast(rhs & (word ? 31 : 63)); + if (word) { + uint32_t value = static_cast(lhs); + uint32_t shifted = + left ? static_cast(value << amount) + : arithmetic ? static_cast( + static_cast(value) >> amount) + : static_cast(value >> amount); + return sext32(shifted); + } + if (left) + return (lhs << amount) & mask; + if (!arithmetic) + return (lhs >> amount) & mask; + return llvm::APInt(64, lhs).ashr(amount).getZExtValue(); + }); + }; + auto bitfieldExtractValue = [&](uint64_t value, uint64_t fieldWidth, + uint64_t offset, bool signedResult) { + if (fieldWidth == 0 || fieldWidth > 64) + return uint64_t{0}; + offset &= 63; + uint64_t rotated = + offset == 0 ? value : ((value >> offset) | (value << (64 - offset))); + uint64_t fieldMask = + fieldWidth == 64 ? ~uint64_t{0} : ((uint64_t{1} << fieldWidth) - 1); + uint64_t resultValue = rotated & fieldMask; + if (signedResult && fieldWidth < 64 && + (resultValue & (uint64_t{1} << (fieldWidth - 1)))) + resultValue |= ~fieldMask; + return resultValue; + }; + auto bitfieldUnary = [&](auto &&fn) { + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 1), + operandConstraint(operands, 2), + [&](uint64_t value, uint64_t fieldWidth, uint64_t offset) { + return fn(value, fieldWidth, offset); + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + }; if (isa(operation)) return noWrapInterval("add"); if (isa(operation)) return noWrapInterval("sub"); if (isa(operation)) return noWrapInterval("mul"); + if (isa(operation)) + return defaultConstraint(resultType); if (isa(operation)) return boundedBinary( [&](uint64_t lhs, uint64_t rhs) { return rhs == 0 ? 0 : lhs / rhs; }); @@ -454,6 +536,203 @@ inferConstraint(Operation *operation, unsigned resultIndex, return boundedBinary([&](uint64_t lhs, uint64_t rhs) { return rhs >= *width ? uint64_t{0} : lhs >> rhs; }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs + rhs; }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs - rhs; }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs & rhs; }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs | rhs; }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs ^ rhs; }); + if (isa(operation)) + return aluShift(false, false, true); + if (isa(operation)) + return aluShift(false, false, false); + if (isa(operation)) + return aluShift(true, false, false); + if (isa(operation)) + return aluShift(false, true, true); + if (isa(operation)) + return aluShift(false, true, false); + if (isa(operation)) + return aluShift(true, true, false); + if (isa(operation)) + return aluBinary([](uint64_t lhs, uint64_t rhs) { + return llvm::APInt(64, lhs).slt(llvm::APInt(64, rhs)) ? lhs : rhs; + }); + if (isa(operation)) + return aluBinary( + [](uint64_t lhs, uint64_t rhs) { return std::min(lhs, rhs); }); + if (isa(operation)) + return aluBinary([](uint64_t lhs, uint64_t rhs) { + return llvm::APInt(64, lhs).sgt(llvm::APInt(64, rhs)) ? lhs : rhs; + }); + if (isa(operation)) + return aluBinary( + [](uint64_t lhs, uint64_t rhs) { return std::max(lhs, rhs); }); + if (isa(operation)) + return aluWordBinary([](uint64_t lhs, uint64_t rhs) { return lhs * rhs; }); + if (isa(operation)) { + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 1), + operandConstraint(operands, 2), + [](uint64_t lhs, uint64_t rhs, uint64_t aux) { + return lhs * rhs + aux; + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + } + if (isa(operation)) { + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 1), + operandConstraint(operands, 2), + [&](uint64_t lhs, uint64_t rhs, uint64_t aux) { + return sext32((lhs * rhs + aux) & 0xffffffffULL); + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + } + if (isa(operation)) { + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 1), + operandConstraint(operands, 2), + [](uint64_t lhs, uint64_t rhs, uint64_t aux) { + return aux - lhs * rhs; + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + } + if (auto extract = dyn_cast(operation)) { + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 1), + operandConstraint(operands, 2), + [&](uint64_t value, uint64_t fieldWidth, uint64_t offset) { + return bitfieldExtractValue(value, fieldWidth, offset, + extract.getSignedMode()); + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + } + if (isa(operation)) + return bitfieldUnary( + [&](uint64_t value, uint64_t fieldWidth, uint64_t offset) { + return llvm::popcount( + bitfieldExtractValue(value, fieldWidth, offset, false)); + }); + if (isa(operation)) + return bitfieldUnary([&](uint64_t value, uint64_t fieldWidth, + uint64_t offset) { + uint64_t field = bitfieldExtractValue(value, fieldWidth, offset, false); + if (fieldWidth == 0 || fieldWidth > 64) + return uint64_t{0}; + if (field == 0) + return fieldWidth; + uint64_t count = 0; + for (uint64_t bit = fieldWidth; + bit != 0 && ((field >> (bit - 1)) & 1) == 0; --bit) + ++count; + return count; + }); + if (isa(operation)) + return bitfieldUnary([&](uint64_t value, uint64_t fieldWidth, + uint64_t offset) { + uint64_t field = bitfieldExtractValue(value, fieldWidth, offset, false); + if (fieldWidth == 0 || fieldWidth > 64) + return uint64_t{0}; + if (field == 0) + return fieldWidth; + uint64_t count = 0; + while (((field >> count) & 1) == 0) + ++count; + return count; + }); + if (isa(operation)) + return bitfieldUnary([&](uint64_t value, uint64_t fieldWidth, + uint64_t offset) { + if (fieldWidth == 0 || fieldWidth > 64) + return value; + unsigned amount = static_cast(offset & 63); + uint64_t rotated = + amount == 0 ? value : ((value >> amount) | (value << (64 - amount))); + uint64_t fieldMask = + fieldWidth == 64 ? ~uint64_t{0} : ((uint64_t{1} << fieldWidth) - 1); + if (isa(operation)) + rotated |= fieldMask; + else + rotated &= ~fieldMask; + return amount == 0 ? rotated + : ((rotated << amount) | (rotated >> (64 - amount))); + }); + if (isa(operation)) + return bitfieldUnary([&](uint64_t value, uint64_t fieldWidth, + uint64_t offset) { + if (fieldWidth == 0 || fieldWidth > 64 || (fieldWidth % 8) != 0) + return uint64_t{0}; + uint64_t field = bitfieldExtractValue(value, fieldWidth, offset, false); + uint64_t resultValue = 0; + for (uint64_t index = 0; index < fieldWidth / 8; ++index) + resultValue |= ((field >> (index * 8)) & 0xffULL) + << ((fieldWidth / 8 - index - 1) * 8); + return resultValue; + }); + if (isa(operation)) { + auto sourceValues = exactValues(operandConstraint(operands, 1)); + if (!sourceValues || sourceValues->size() != 1) + return defaultConstraint(resultType); + ValueConstraint result = evaluateTernary( + operandConstraint(operands, 0), operandConstraint(operands, 2), + operandConstraint(operands, 3), + [&](uint64_t base, uint64_t fieldWidth, uint64_t offset) { + if (fieldWidth == 0 || fieldWidth > 64) + return uint64_t{0}; + uint64_t resultValue = base; + for (uint64_t bit = 0; bit < fieldWidth; ++bit) { + uint64_t destination = (offset + bit) & 63; + uint64_t bitMask = uint64_t{1} << destination; + resultValue = (resultValue & ~bitMask) | + (((sourceValues->front() >> bit) & 1) ? bitMask : 0); + } + return resultValue; + }); + return result.kind == ValueConstraintKind::Unknown + ? defaultConstraint(resultType) + : result; + } + if (isa(operation)) { + auto values = exactValues(operandConstraint(operands, 0)); + auto widths = exactValues(operandConstraint(operands, 1)); + if (values && widths && values->size() == 1 && widths->size() == 1 && + widths->front() > 0 && widths->front() <= 64) { + uint64_t fieldWidth = widths->front(); + uint64_t fieldMask = + fieldWidth == 64 ? ~uint64_t{0} : ((uint64_t{1} << fieldWidth) - 1); + uint64_t resultValue = values->front() & fieldMask; + if (isa(operation) && fieldWidth < 64 && + (resultValue & (uint64_t{1} << (fieldWidth - 1)))) + resultValue |= ~fieldMask; + return ValueConstraint::constant(resultValue); + } + return defaultConstraint(resultType); + } + if (isa(operation)) { + auto predicate = exactValues(operandConstraint(operands, 0)); + auto lhs = exactValues(operandConstraint(operands, 1)); + auto rhs = exactValues(operandConstraint(operands, 2)); + auto negate = exactValues(operandConstraint(operands, 3)); + if (predicate && lhs && rhs && negate && predicate->size() == 1 && + lhs->size() == 1 && rhs->size() == 1 && negate->size() == 1) + return ValueConstraint::constant( + predicate->front() + ? lhs->front() + : (negate->front() ? uint64_t{0} - rhs->front() : rhs->front())); + return defaultConstraint(resultType); + } if (isa(operation)) { ValueConstraint result = evaluateUnary(operandConstraint(operands, 0), @@ -536,9 +815,8 @@ inferConstraint(Operation *operation, unsigned resultIndex, uint64_t entries = 1; bool valid = !shape.asArrayRef().empty(); for (int64_t extent : shape.asArrayRef()) { - if (extent <= 0 || - entries > std::numeric_limits::max() / - static_cast(extent)) { + if (extent <= 0 || entries > std::numeric_limits::max() / + static_cast(extent)) { valid = false; break; } @@ -1134,7 +1412,8 @@ ACDataFlowAnalyzer::stateSnapshots(Operation *scope) const { continue; } if (auto get = dyn_cast(definition)) { - children.push_back({get.getRecord(), setSource, {get.getField().str()}}); + children.push_back( + {get.getRecord(), setSource, {get.getField().str()}}); flushChildren(); continue; } @@ -1155,17 +1434,15 @@ ACDataFlowAnalyzer::stateSnapshots(Operation *scope) const { } if (auto choose = dyn_cast(definition)) { if (!choose.getKey().empty()) { - Type entryType = - cast( - choose.getKey().front().getArgument(0).getType()) - .getElementType(); + Type entryType = cast( + choose.getKey().front().getArgument(0).getType()) + .getElementType(); addSnapshot(choose.getTable(), {}, {}, "all", predicate, completeStateFields(choose, entryType)); } children.push_back({choose.getMask(), {}, {}}); - Value firstIndex = choose.getResults().empty() - ? Value() - : choose.getResults().front(); + Value firstIndex = + choose.getResults().empty() ? Value() : choose.getResults().front(); for (Block &block : choose.getKey()) for (Value operand : block.getTerminator()->getOperands()) children.push_back({operand, firstIndex, {}}); diff --git a/compiler/acir/lib/CodeGen/QueueGraphGenerator.cpp b/compiler/acir/lib/CodeGen/QueueGraphGenerator.cpp index d468ddbb2..0407647b8 100644 --- a/compiler/acir/lib/CodeGen/QueueGraphGenerator.cpp +++ b/compiler/acir/lib/CodeGen/QueueGraphGenerator.cpp @@ -33,7 +33,7 @@ llvm::Error generatorError(const llvm::Twine &message) { template void appendInitializer(std::vector &initializers, - const Values &...values) { + const Values &... values) { std::string initializer; llvm::raw_string_ostream output(initializer); (output << ... << values); @@ -724,6 +724,7 @@ emitExpressionBody(const QueueGraphPlan &plan, const QueueBlockPlan &block, std::string padding(indent, ' '); llvm::StringMap priorityEncodings; llvm::StringMap helperCallValues; + llvm::StringMap divremResults; llvm::StringMap> tableChoices; llvm::StringMap rangeCheckedValues; llvm::StringMap> priorChoiceIndices; @@ -1288,6 +1289,40 @@ emitExpressionBody(const QueueGraphPlan &plan, const QueueBlockPlan &block, << first->str() << ");\n"; continue; } + if (expression.kind == "divrem_quotient" || + expression.kind == "divrem_remainder") { + if (expression.operands.size() != 4) + return generatorError("divrem expression arity mismatch"); + const std::string key = + expression.operands[0] + ":" + expression.operands[1] + ":" + + expression.operands[2] + ":" + expression.operands[3]; + auto [found, inserted] = divremResults.try_emplace( + key, "divrem_" + identifier(expression.operands[0]) + "_" + + identifier(expression.operands[1]) + "_" + + identifier(expression.operands[2]) + "_" + + identifier(expression.operands[3])); + if (inserted) { + output << padding << "auto " << found->getValue() << " = gfsim::divrem(" + << first->str() << ", "; + auto second = operand(1); + if (!second) + return second.takeError(); + auto signedMode = operand(2); + if (!signedMode) + return signedMode.takeError(); + auto wordMode = operand(3); + if (!wordMode) + return wordMode.takeError(); + output << second->str() << ", static_cast(" << signedMode->str() + << "), static_cast(" << wordMode->str() << "));\n"; + } + output << padding << "auto " << expression.result << " = " + << found->getValue() << "." + << (expression.kind == "divrem_quotient" ? "quotient" + : "remainder") + << ";\n"; + continue; + } if (expression.kind == "table_choose_index" || expression.kind == "table_choose_valid") { QueueBlockPlan nested; @@ -1644,6 +1679,107 @@ emitExpressionBody(const QueueGraphPlan &plan, const QueueBlockPlan &block, << ";\n"; continue; } + if (llvm::StringSwitch(expression.kind) + .Cases({"addw", "subw", "andw", "orw", "xorw", "sll", "srl", "sra", + "sllw", "srlw", "sraw", "smin", "umin", "smax", "umax", + "mulw"}, + true) + .Default(false)) { + auto second = operand(1); + if (!second) + return second.takeError(); + output << padding << "auto " << expression.result + << " = gfsim::" << expression.kind << "(" << first->str() << ", " + << second->str() << ");\n"; + continue; + } + if (llvm::StringSwitch(expression.kind) + .Cases({"madd", "maddw", "msub"}, true) + .Default(false)) { + auto second = operand(1); + auto third = operand(2); + if (!second) + return second.takeError(); + if (!third) + return third.takeError(); + output << padding << "auto " << expression.result + << " = gfsim::" << expression.kind << "(" << first->str() << ", " + << second->str() << ", " << third->str() << ");\n"; + continue; + } + if (expression.kind == "bitfield_extract" || + expression.kind == "bitfield_popcount" || + expression.kind == "bitfield_clz" || + expression.kind == "bitfield_ctz" || + expression.kind == "bitfield_clear" || + expression.kind == "bitfield_set" || + expression.kind == "bitfield_reverse_bytes") { + auto width = operand(1); + auto offset = operand(2); + if (!width) + return width.takeError(); + if (!offset) + return offset.takeError(); + const llvm::StringRef function = + llvm::StringSwitch(expression.kind) + .Case("bitfield_extract", "bitfieldExtract") + .Case("bitfield_popcount", "bitfieldPopcount") + .Case("bitfield_clz", "bitfieldClz") + .Case("bitfield_ctz", "bitfieldCtz") + .Case("bitfield_clear", "bitfieldClear") + .Case("bitfield_set", "bitfieldSet") + .Case("bitfield_reverse_bytes", "bitfieldReverseBytes") + .Default(""); + if (function.empty()) + return generatorError("unknown ALU bitfield expression kind"); + output << padding << "auto " << expression.result + << " = gfsim::" << function.str() << "(" << first->str() << ", " + << width->str() << ", " << offset->str(); + if (expression.kind == "bitfield_extract") + output << ", " << (expression.predicate == "signed" ? "true" : "false"); + output << ");\n"; + continue; + } + if (expression.kind == "bitfield_insert") { + auto source = operand(1); + auto width = operand(2); + auto offset = operand(3); + if (!source) + return source.takeError(); + if (!width) + return width.takeError(); + if (!offset) + return offset.takeError(); + output << padding << "auto " << expression.result + << " = gfsim::bitfieldInsert(" << first->str() << ", " + << source->str() << ", " << width->str() << ", " << offset->str() + << ");\n"; + continue; + } + if (expression.kind == "sext_low" || expression.kind == "zext_low") { + auto width = operand(1); + if (!width) + return width.takeError(); + output << padding << "auto " << expression.result << " = gfsim::" + << (expression.kind == "sext_low" ? "sextLow" : "zextLow") << "(" + << first->str() << ", " << width->str() << ");\n"; + continue; + } + if (expression.kind == "csel") { + auto lhs = operand(1); + auto rhs = operand(2); + auto negate = operand(3); + if (!lhs) + return lhs.takeError(); + if (!rhs) + return rhs.takeError(); + if (!negate) + return negate.takeError(); + output << padding << "auto " << expression.result << " = gfsim::csel(" + << first->str() << ", " << lhs->str() << ", " << rhs->str() << ", " + << negate->str() << ");\n"; + continue; + } auto second = operand(1); if (!second) return second.takeError(); @@ -1670,9 +1806,9 @@ emitExpressionBody(const QueueGraphPlan &plan, const QueueBlockPlan &block, operation = "-"; else if (expression.kind == "mul") operation = "*"; - else if (expression.kind == "udiv") + else if (expression.kind == "udiv" || expression.kind == "sdiv") operation = "/"; - else if (expression.kind == "urem") + else if (expression.kind == "urem" || expression.kind == "srem") operation = "%"; else if (expression.kind == "and") operation = "&"; @@ -1712,6 +1848,12 @@ emitExpressionBody(const QueueGraphPlan &plan, const QueueBlockPlan &block, output << first->str() << ' ' << operation.str() << ' ' << second->str(); output << "}"; + } else if (expression.kind == "sdiv") { + output << "gfsim::signedDiv(" << first->str() << ", " << second->str() + << ")"; + } else if (expression.kind == "srem") { + output << "gfsim::signedRem(" << first->str() << ", " << second->str() + << ")"; } else { output << first->str() << ' ' << operation.str() << ' ' << second->str(); } @@ -2900,8 +3042,11 @@ generateStructuredQueueGraphCpp(const QueueGraphPlan &plan) { output << "// Generated from hierarchy-preserving frozen ACIR QueueGraph " "plan; do not edit.\n" "#include \"gfsim/bits.h\"\n" + "#include \"gfsim/alu.h\"\n" + "#include \"gfsim/bitfield.h\"\n" "#include \"gfsim/dispatch.h\"\n" "#include \"gfsim/object.h\"\n" + "#include \"gfsim/divrem.h\"\n" "#include \"gfsim/priority_encode.h\"\n" "#include \"gfsim/queue.h\"\n" "#include \"gfsim/queue_blocks.h\"\n\n" @@ -3196,9 +3341,9 @@ generateStructuredQueueGraphCpp(const QueueGraphPlan &plan) { } tupleResult.append(", ").append(firing.guard).push_back('}'); const std::string &primaryValue = - !firing.stateWrites.empty() ? firing.stateWrites.front().index - : !firing.yields.empty() ? firing.yields.front() - : firing.guard; + !firing.stateWrites.empty() + ? firing.stateWrites.front().index + : !firing.yields.empty() ? firing.yields.front() : firing.guard; auto body = emitExpressionBody(specialization, evaluation, primaryValue, 6, true, false, additional, tupleResult); if (!body) @@ -4537,9 +4682,12 @@ llvm::Expected generateQueueGraphCpp(const QueueGraphPlan &plan) { if (!plan.specializationFingerprint.empty()) output << "// Specialization: " << plan.specializationFingerprint << "\n"; output << "#include \"gfsim/bits.h\"\n" + "#include \"gfsim/alu.h\"\n" + "#include \"gfsim/bitfield.h\"\n" "#include \"gfsim/dispatch.h\"\n" "#include \"gfsim/object.h\"\n" "#include \"gfsim/count_zeros.h\"\n" + "#include \"gfsim/divrem.h\"\n" "#include \"gfsim/popcount.h\"\n" "#include \"gfsim/priority_encode.h\"\n" "#include \"gfsim/queue.h\"\n" @@ -4642,10 +4790,11 @@ llvm::Expected generateQueueGraphCpp(const QueueGraphPlan &plan) { output << ">;\n" << "inline constexpr auto " << identifier(selection.name) << "_choose_policy = gfsim::TableChoosePolicy::" - << (selection.policy == "first" ? "First" - : selection.policy == "min" ? "Min" - : selection.policy == "max" ? "Max" - : "RoundRobin") + << (selection.policy == "first" + ? "First" + : selection.policy == "min" + ? "Min" + : selection.policy == "max" ? "Max" : "RoundRobin") << ";\n" << "inline constexpr auto " << identifier(selection.name) << "_key_ordering = gfsim::TableKeyOrdering::" @@ -4803,9 +4952,9 @@ llvm::Expected generateQueueGraphCpp(const QueueGraphPlan &plan) { appendTupleValue(block->guard); tupleResult.push_back('}'); const std::string &primaryValue = - !block->stateWrites.empty() ? block->stateWrites.front().index - : !block->yields.empty() ? block->yields.front() - : block->guard; + !block->stateWrites.empty() + ? block->stateWrites.front().index + : !block->yields.empty() ? block->yields.front() : block->guard; auto evaluationBody = emitExpressionBody(plan, evaluation, primaryValue, 6, true, false, additional, tupleResult); @@ -5106,9 +5255,9 @@ llvm::Expected generateQueueGraphCpp(const QueueGraphPlan &plan) { tupleResult.push_back('}'); auto evaluationBody = emitExpressionBody( plan, evaluation, - !block->stateWrites.empty() ? block->stateWrites.front().index - : !block->yields.empty() ? block->yields.front() - : block->guard, + !block->stateWrites.empty() + ? block->stateWrites.front().index + : !block->yields.empty() ? block->yields.front() : block->guard, 6, true, false, additional, tupleResult); if (!evaluationBody) return evaluationBody.takeError(); @@ -5264,9 +5413,10 @@ llvm::Expected generateQueueGraphCpp(const QueueGraphPlan &plan) { const std::vector policyNames = block->kind == "table_read" ? std::vector{"address", "when"} - : block->kind == "table_masked_write" - ? std::vector{"mask", "enable", "value"} - : std::vector{"address", "enable", "value"}; + : block->kind == "table_masked_write" + ? std::vector{"mask", "enable", "value"} + : std::vector{"address", "enable", + "value"}; for (auto [policyIndex, policyName] : llvm::enumerate(policyNames)) { llvm::StringRef resultType = table->entryType; if (block->yields[policyIndex] != "item") { diff --git a/compiler/acir/lib/CodeGen/QueueGraphPlan.cpp b/compiler/acir/lib/CodeGen/QueueGraphPlan.cpp index bb19593d4..1cc01e26e 100644 --- a/compiler/acir/lib/CodeGen/QueueGraphPlan.cpp +++ b/compiler/acir/lib/CodeGen/QueueGraphPlan.cpp @@ -315,8 +315,7 @@ mlirValueBitWidth(mlir::Operation *from, mlir::Type type, return finish(planError("QueueGraph value type has no bit-width model")); } -std::optional> -rangeBounds(llvm::StringRef type) { +std::optional> rangeBounds(llvm::StringRef type) { constexpr llvm::StringLiteral prefix = "!ac.range<"; if (!type.starts_with(prefix) || !type.ends_with('>')) return std::nullopt; @@ -595,8 +594,7 @@ inferPlanConstraint(const QueueExpressionPlan &expression, return ValueConstraint::closedInterval(0, right.values.front() & mask); } if (expression.kind == "urem" && - right.kind == ValueConstraintKind::Constant && - right.values.front() != 0) + right.kind == ValueConstraintKind::Constant && right.values.front() != 0) return ValueConstraint::closedInterval(0, right.values.front() - 1); if (expression.kind == "not" && left.kind == ValueConstraintKind::Constant) return ValueConstraint::constant((~left.values.front()) & mask); @@ -736,23 +734,23 @@ extractSourceProvenance(mlir::Operation *operation) { return planError("source provenance must be a non-empty origin array"); for (mlir::Attribute rawOrigin : origins) { auto origin = mlir::dyn_cast(rawOrigin); - auto frames = origin ? origin.getAs("frames") - : mlir::ArrayAttr(); + auto frames = + origin ? origin.getAs("frames") : mlir::ArrayAttr(); if (!origin || origin.size() != 1 || !frames || frames.empty()) return planError("source provenance origin is malformed"); QueueSourceOriginPlan plannedOrigin; for (mlir::Attribute rawFrame : frames) { auto frame = mlir::dyn_cast(rawFrame); - auto file = frame ? frame.getAs("file") - : mlir::StringAttr(); - auto kind = frame ? frame.getAs("kind") - : mlir::StringAttr(); - auto line = frame ? frame.getAs("line") - : mlir::IntegerAttr(); + auto file = + frame ? frame.getAs("file") : mlir::StringAttr(); + auto kind = + frame ? frame.getAs("kind") : mlir::StringAttr(); + auto line = + frame ? frame.getAs("line") : mlir::IntegerAttr(); auto column = frame ? frame.getAs("column") : mlir::IntegerAttr(); - auto symbol = frame ? frame.getAs("symbol") - : mlir::StringAttr(); + auto symbol = + frame ? frame.getAs("symbol") : mlir::StringAttr(); if (!frame || (frame.size() != 4 && frame.size() != 5) || !file || !kind || !line || !column || (frame.size() == 5) != static_cast(symbol)) @@ -764,8 +762,7 @@ extractSourceProvenance(mlir::Operation *operation) { if ((rawKind != "statement" && rawKind != "definition" && rawKind != "inline_callsite" && rawKind != "instance" && rawKind != "specialization") || - !isValidPythonSourcePath(rawFile) || - rawLine <= 0 || rawColumn <= 0) + !isValidPythonSourcePath(rawFile) || rawLine <= 0 || rawColumn <= 0) return planError("source provenance frame is malformed"); plannedOrigin.push_back( {rawKind.str(), rawFile.str(), static_cast(rawLine), @@ -845,10 +842,11 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, if (!argumentNames.empty() && argumentNames.size() != block.getNumArguments()) return planError("helper argument identity count is malformed"); for (auto [index, argument] : llvm::enumerate(block.getArguments())) { - values[argument] = !argumentNames.empty() ? argumentNames[index] - : index == 0 ? (prefix == "v" ? "item" : "entry") - : (prefix == "v" ? "item" : "entry") + - std::to_string(index); + values[argument] = !argumentNames.empty() + ? argumentNames[index] + : index == 0 ? (prefix == "v" ? "item" : "entry") + : (prefix == "v" ? "item" : "entry") + + std::to_string(index); identities.insert(values[argument]); } auto resultIdentity = [&](mlir::Operation &operation, @@ -897,11 +895,15 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, std::string result = resultIdentity( operation, prefix.str() + std::to_string(plan.expressions.size())); values[operation.getResult(0)] = result; - QueueExpressionPlan expression{ - std::move(result), kind.str(), printType(resultType.getElementType()), - std::move(*operands), field.str(), predicate.str(), literal.str()}; - if (auto target = operation.getAttrOfType( - "ac.static_type_target")) + QueueExpressionPlan expression{std::move(result), + kind.str(), + printType(resultType.getElementType()), + std::move(*operands), + field.str(), + predicate.str(), + literal.str()}; + if (auto target = + operation.getAttrOfType("ac.static_type_target")) expression.staticTypeTarget = target.getValue().str(); expression.sourceProvenance = currentExpressionProvenance; plan.expressions.push_back(std::move(expression)); @@ -949,7 +951,8 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, } if (auto constant = mlir::dyn_cast(operation)) { std::string literal = printAttribute(constant.getValueAttr()); - if (auto integer = mlir::dyn_cast(constant.getValueAttr()); + if (auto integer = + mlir::dyn_cast(constant.getValueAttr()); integer && mlir::cast(integer.getType()).getWidth() > 1 && mlir::cast(integer.getType()).getWidth() <= 64) @@ -1097,8 +1100,7 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, active.clear(); auto indexWidth = mlirValueBitWidth( update, - mlir::cast(update.getIndex().getType()) - .getElementType(), + mlir::cast(update.getIndex().getType()).getElementType(), active); if (!indexWidth) return indexWidth.takeError(); @@ -1128,11 +1130,56 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, return error; continue; } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sdiv")) + return error; + continue; + } if (mlir::isa(operation)) { if (auto error = append(operation, "urem")) return error; continue; } + if (mlir::isa(operation)) { + if (auto error = append(operation, "srem")) + return error; + continue; + } + if (auto divrem = mlir::dyn_cast(operation)) { + if (operation.getNumResults() != 2) + return planError("divrem must produce quotient and remainder"); + auto resultTypes = + mlir::dyn_cast(operation.getResult(0).getType()); + auto remainderType = + mlir::dyn_cast(operation.getResult(1).getType()); + if (!resultTypes || !remainderType || + resultTypes.getElementType() != remainderType.getElementType()) + return planError("divrem result types must match"); + auto operands = operandNames(operation.getOperands()); + if (!operands) + return operands.takeError(); + if (operands->size() != 4) + return planError("divrem operand arity mismatch"); + std::string quotient = + prefix.str() + std::to_string(plan.expressions.size()); + std::string remainder = + prefix.str() + std::to_string(plan.expressions.size() + 1); + plan.expressions.push_back({quotient, + "divrem_quotient", + printType(resultTypes.getElementType()), + *operands, + {}, + {}}); + plan.expressions.push_back({remainder, + "divrem_remainder", + printType(remainderType.getElementType()), + *operands, + {}, + {}}); + values[operation.getResult(0)] = quotient; + values[operation.getResult(1)] = remainder; + continue; + } if (mlir::isa(operation)) { if (auto error = append(operation, "and")) return error; @@ -1158,6 +1205,157 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, return error; continue; } + if (mlir::isa(operation)) { + if (auto error = append(operation, "addw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "subw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "andw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "orw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "xorw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sll")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "srl")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sra")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sllw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "srlw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sraw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "smin")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "umin")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "smax")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "umax")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "mulw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "madd")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "maddw")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "msub")) + return error; + continue; + } + if (auto extract = mlir::dyn_cast(operation)) { + if (auto error = append(operation, "bitfield_extract", {}, + extract.getSignedMode() ? "signed" : "unsigned")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_popcount")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_clz")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_ctz")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_clear")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_set")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_reverse_bytes")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "bitfield_insert")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "sext_low")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "zext_low")) + return error; + continue; + } + if (mlir::isa(operation)) { + if (auto error = append(operation, "csel")) + return error; + continue; + } if (auto matches = mlir::dyn_cast(operation)) { if (auto error = append(operation, "masked_match")) return error; @@ -1189,10 +1387,13 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, std::string result = prefix.str() + std::to_string(plan.expressions.size()); values[resultValue] = result; - QueueExpressionPlan expression{ - std::move(result), kind.str(), - printType(resultType.getElementType()), *operands, {}, - priority.getOrder().str(), {}}; + QueueExpressionPlan expression{std::move(result), + kind.str(), + printType(resultType.getElementType()), + *operands, + {}, + priority.getOrder().str(), + {}}; expression.sourceProvenance = currentExpressionProvenance; plan.expressions.push_back(std::move(expression)); } @@ -1245,9 +1446,8 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, auto operands = operandNames(checked->getOperands()); if (!operands) return operands.takeError(); - const std::string group = - prefix.str() + "range_checked_" + - std::to_string(plan.expressions.size()); + const std::string group = prefix.str() + "range_checked_" + + std::to_string(plan.expressions.size()); const std::array, 2> results = {{ {checked.getValue(), "range_checked_value"}, {checked.getValid(), "range_checked_valid"}, @@ -1257,9 +1457,9 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, std::string result = prefix.str() + std::to_string(plan.expressions.size()); values[resultValue] = result; - QueueExpressionPlan expression{ - result, kind.str(), printType(resultType.getElementType()), - *operands}; + QueueExpressionPlan expression{result, kind.str(), + printType(resultType.getElementType()), + *operands}; expression.field = printType(mlir::cast(checked.getValue().getType()) .getElementType()); @@ -1293,8 +1493,8 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, continue; } if (auto compare = mlir::dyn_cast(operation)) { - if (auto error = append(operation, "range_cmp", {}, - compare.getPredicate())) + if (auto error = + append(operation, "range_cmp", {}, compare.getPredicate())) return error; continue; } @@ -1346,13 +1546,11 @@ extractExpressions(mlir::Region ®ion, QueueBlockPlan &plan, } continue; } - if (auto release = - mlir::dyn_cast(operation)) { + if (auto release = mlir::dyn_cast(operation)) { auto when = values.find(release.getWhen()); if (when == values.end()) return planError("slot release guard is not a known firing value"); - plan.slotReleases.push_back( - {release.getSlot().str(), when->second}); + plan.slotReleases.push_back({release.getSlot().str(), when->second}); continue; } if (auto match = mlir::dyn_cast(operation)) { @@ -2189,8 +2387,8 @@ void materializeCaptureOnlySlots(QueueGraphPlan &plan) { return block.kind == "slot" && block.slot == slot.name; })) continue; - QueueBlockPlan capture{"slot", slot.name + "__capture", slot.scope, - {slot.input}, {}}; + QueueBlockPlan capture{ + "slot", slot.name + "__capture", slot.scope, {slot.input}, {}}; capture.lexicalOrder = plan.blocks.size() + plan.moduleInstances.size(); capture.slot = slot.name; capture.yields = {"release_disabled"}; @@ -2945,7 +3143,9 @@ class Extractor { slot.getSymName().str(), printType(mlir::cast(slot.getInput().getType()) .getElementType()), - *input, scopePath(scope), slot.getStableId().str(), + *input, + scopePath(scope), + slot.getStableId().str(), slot.getOwner().str()}; slotPlan.sourceProvenance = currentSourceProvenance; plan.slots.push_back(std::move(slotPlan)); @@ -3625,10 +3825,13 @@ class Extractor { return error; outputs.push_back(std::move(name)); } - QueueModuleInstancePlan plannedInstance{ - instance.getSymName().str(), instance.getDefinition().str(), - fingerprint.getValue().str(), scopePath(scope), std::move(*inputs), - std::move(outputs), nextLexicalOrder++}; + QueueModuleInstancePlan plannedInstance{instance.getSymName().str(), + instance.getDefinition().str(), + fingerprint.getValue().str(), + scopePath(scope), + std::move(*inputs), + std::move(outputs), + nextLexicalOrder++}; auto provenance = extractSourceProvenance(instance); if (!provenance) return provenance.takeError(); @@ -3903,7 +4106,8 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { return planError("static config schema or value is invalid"); } if (*canonicalSchema != config.schema || *canonicalValue != config.value) - return planError("static config schema and value must use canonical JSON"); + return planError( + "static config schema and value must use canonical JSON"); auto schema = llvm::json::parse(*canonicalSchema); auto value = llvm::json::parse(*canonicalValue); if (!schema || !value || !verifyStaticConfigValue(*schema, *value)) @@ -3918,7 +4122,8 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { ++configProjectionMatches[name]; auto projected = projectStaticConfigInteger(*schema, *value, path); if (!projected || *projected != bindingValue) - return planError("static config projection disagrees with its root binding"); + return planError( + "static config projection disagrees with its root binding"); usedConfigRoots.insert(config.root); } } @@ -3967,10 +4172,13 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { return {true, std::nullopt}; if (!plan.interfaceOutputs.empty()) return ordinal < plan.interfaceOutputs.size() - ? std::pair>{ - true, plan.interfaceOutputs[ordinal].payloadType} - : std::pair>{ - true, std::nullopt}; + ? std::pair>{true, + plan.interfaceOutputs + [ordinal] + .payloadType} + : std::pair>{true, + std::nullopt}; llvm::SmallVector outputs; for (const QueueBlockPlan &block : plan.blocks) if (block.kind == "sink" && block.inputs.size() == 1) @@ -4016,8 +4224,8 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { return {true, std::nullopt}; return {true, ports[ordinal].payloadType}; }; - auto resolveExpressionType = [&](llvm::StringRef path) - -> std::optional { + auto resolveExpressionType = + [&](llvm::StringRef path) -> std::optional { std::optional resolved; bool conflict = false; auto visit = [&](auto &&self, const auto &expressions) -> void { @@ -4026,8 +4234,7 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { llvm::StringRef candidate = expression.type; if (expression.kind == "range_checked_valid") candidate = expression.field; - if (!rangeBounds(candidate) || - (resolved && *resolved != candidate)) + if (!rangeBounds(candidate) || (resolved && *resolved != candidate)) conflict = true; else resolved = candidate.str(); @@ -4088,10 +4295,11 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { int64_t right = stack.pop_back_val(); int64_t left = stack.pop_back_val(); int64_t result = 0; - bool overflow = rawToken == "add" ? llvm::AddOverflow(left, right, result) - : rawToken == "sub" - ? llvm::SubOverflow(left, right, result) - : llvm::MulOverflow(left, right, result); + bool overflow = rawToken == "add" + ? llvm::AddOverflow(left, right, result) + : rawToken == "sub" + ? llvm::SubOverflow(left, right, result) + : llvm::MulOverflow(left, right, result); if (overflow) return planError("static type check arithmetic overflow"); stack.push_back(result); @@ -4104,10 +4312,10 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { std::string ownedResolvedType = check.concreteType; llvm::StringRef resolvedType = ownedResolvedType; if (!check.concreteType.empty()) { - auto [applicable, actual] = path.starts_with("expression.") - ? std::pair{true, - resolveExpressionType(path)} - : resolveInterfaceType(path); + auto [applicable, actual] = + path.starts_with("expression.") + ? std::pair{true, resolveExpressionType(path)} + : resolveInterfaceType(path); if (applicable && (!actual || *actual != check.concreteType)) return planError( "static interface type check disagrees with the actual endpoint '" + @@ -4169,11 +4377,11 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { auto bounds = rangeBounds(resolvedType); uint64_t expected = 0; if (bounds) - expected = kind == "range_lower" - ? bounds->first - : bounds->second + - (bounds->second != - std::numeric_limits::max()); + expected = + kind == "range_lower" + ? bounds->first + : bounds->second + + (bounds->second != std::numeric_limits::max()); if (!bounds || check.result < 0 || static_cast(check.result) != expected) return planError("static bounded range is inconsistent"); @@ -4203,8 +4411,8 @@ llvm::Error verifyStaticTypeMetadata(const QueueGraphPlan &plan) { return llvm::Error::success(); }; for (const QueueBlockPlan &block : plan.blocks) - if (auto error = verifyExpressionTargets(verifyExpressionTargets, - block.expressions)) + if (auto error = + verifyExpressionTargets(verifyExpressionTargets, block.expressions)) return error; for (const QueueHelperPlan &helper : plan.helpers) if (auto error = verifyExpressionTargets(verifyExpressionTargets, @@ -4506,14 +4714,46 @@ bool isEffectFreeTableMatchExpression(const QueueExpressionPlan &expression) { true) .Cases({"tuple_create", "array_create", "record_create", "bit_insert"}, true) - .Cases({"with", "add", "sub", "mul", "udiv", "urem"}, true) + .Cases({"with", "add", "sub", "mul", "udiv", "sdiv", "urem", "srem"}, + true) .Cases({"and", "or", "xor", "shl", "shr"}, true) + .Cases({"addw", + "subw", + "andw", + "orw", + "xorw", + "sll", + "srl", + "sra", + "sllw", + "srlw", + "sraw", + "smin", + "umin", + "smax", + "umax", + "mulw", + "madd", + "maddw", + "msub", + "bitfield_extract", + "bitfield_popcount", + "bitfield_clz", + "bitfield_ctz", + "bitfield_clear", + "bitfield_set", + "bitfield_reverse_bytes", + "bitfield_insert", + "sext_low", + "zext_low", + "csel"}, + true) .Cases({"priority_index", "priority_valid"}, true) .Cases({"range_wrap", "range_saturate", "range_refine", "range_bits", "range_add", "range_sub", "range_cmp"}, true) - .Cases({"range_checked_value", "range_checked_valid", - "array_get_dynamic", "array_update_dynamic"}, + .Cases({"range_checked_value", "range_checked_valid", "array_get_dynamic", + "array_update_dynamic"}, true) .Default(false); } @@ -4541,10 +4781,11 @@ verifySourceProvenancePlan(const QueueSourceProvenancePlan &provenance) { bool firstFrame = true; for (const QueueSourceFramePlan &frame : origin) { unsigned kindRank = - frame.kind == "statement" || frame.kind == "definition" ? 0 - : frame.kind == "inline_callsite" ? 1 - : frame.kind == "instance" ? 2 - : 3; + frame.kind == "statement" || frame.kind == "definition" + ? 0 + : frame.kind == "inline_callsite" + ? 1 + : frame.kind == "instance" ? 2 : 3; if ((frame.kind != "statement" && frame.kind != "definition" && frame.kind != "inline_callsite" && frame.kind != "instance" && frame.kind != "specialization") || @@ -4946,20 +5187,20 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return false; bool supported = false; if (auto name = payloadTypeName(type)) { - auto payload = llvm::find_if( - plan.payloads, [&](const QueuePayloadPlan &candidate) { + auto payload = + llvm::find_if(plan.payloads, [&](const QueuePayloadPlan &candidate) { return candidate.name == *name; }); - supported = - payload != plan.payloads.end() && - llvm::all_of(payload->fields, [&](const QueuePayloadFieldPlan &field) { - return self(self, field.type, active); - }); + supported = payload != plan.payloads.end() && + llvm::all_of(payload->fields, + [&](const QueuePayloadFieldPlan &field) { + return self(self, field.type, active); + }); } else { - auto aggregate = llvm::find_if( - plan.aggregates, [&](const QueueAggregatePlan &candidate) { - return candidate.type == type; - }); + auto aggregate = llvm::find_if(plan.aggregates, + [&](const QueueAggregatePlan &candidate) { + return candidate.type == type; + }); supported = aggregate != plan.aggregates.end() && llvm::all_of(aggregate->elements, [&](llvm::StringRef element) { @@ -4977,8 +5218,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return false; bool found = false; if (auto name = payloadTypeName(type)) { - auto payload = llvm::find_if( - plan.payloads, [&](const QueuePayloadPlan &candidate) { + auto payload = + llvm::find_if(plan.payloads, [&](const QueuePayloadPlan &candidate) { return candidate.name == *name; }); found = payload != plan.payloads.end() && @@ -4987,15 +5228,14 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return self(self, field.type, active); }); } else { - auto aggregate = llvm::find_if( - plan.aggregates, [&](const QueueAggregatePlan &candidate) { - return candidate.type == type; - }); + auto aggregate = llvm::find_if(plan.aggregates, + [&](const QueueAggregatePlan &candidate) { + return candidate.type == type; + }); found = aggregate != plan.aggregates.end() && - llvm::any_of(aggregate->elements, - [&](llvm::StringRef element) { - return self(self, element, active); - }); + llvm::any_of(aggregate->elements, [&](llvm::StringRef element) { + return self(self, element, active); + }); } active.erase(type); return found; @@ -5493,8 +5733,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return false; bool found = false; if (std::optional name = payloadTypeName(type)) { - auto payload = llvm::find_if( - plan.payloads, [&](const QueuePayloadPlan &candidate) { + auto payload = + llvm::find_if(plan.payloads, [&](const QueuePayloadPlan &candidate) { return candidate.name == *name; }); if (payload != plan.payloads.end()) @@ -5502,10 +5742,10 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return self(self, field.type, active); }); } else { - auto aggregate = llvm::find_if( - plan.aggregates, [&](const QueueAggregatePlan &candidate) { - return candidate.type == type; - }); + auto aggregate = llvm::find_if(plan.aggregates, + [&](const QueueAggregatePlan &candidate) { + return candidate.type == type; + }); if (aggregate != plan.aggregates.end()) found = llvm::any_of(aggregate->elements, [&](llvm::StringRef element) { return self(self, element, active); @@ -5677,8 +5917,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { rangeBounds(operand->getValue()).has_value() && rangeBounds(operand->getValue())->second < extent; if (operand == valueTypes.end() || - (!bounded && operand->getValue() != - "i" + std::to_string(expectedWidth))) + (!bounded && + operand->getValue() != "i" + std::to_string(expectedWidth))) return planError("Table coordinate type is inconsistent"); } const std::string expectedType = @@ -5824,12 +6064,12 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { auto name = source == valueTypes.end() ? std::optional() : payloadTypeName(source->getValue()); - auto payload = name ? llvm::find_if( - plan.payloads, - [&](const QueuePayloadPlan &candidate) { - return candidate.name == *name; - }) - : plan.payloads.end(); + auto payload = + name ? llvm::find_if(plan.payloads, + [&](const QueuePayloadPlan &candidate) { + return candidate.name == *name; + }) + : plan.payloads.end(); const QueuePayloadFieldPlan *field = nullptr; if (payload != plan.payloads.end()) { auto found = llvm::find_if(payload->fields, [&](const auto &item) { @@ -5850,12 +6090,12 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { auto name = base == valueTypes.end() ? std::optional() : payloadTypeName(base->getValue()); - auto payload = name ? llvm::find_if( - plan.payloads, - [&](const QueuePayloadPlan &candidate) { - return candidate.name == *name; - }) - : plan.payloads.end(); + auto payload = + name ? llvm::find_if(plan.payloads, + [&](const QueuePayloadPlan &candidate) { + return candidate.name == *name; + }) + : plan.payloads.end(); const QueuePayloadFieldPlan *field = nullptr; if (payload != plan.payloads.end()) { auto found = llvm::find_if(payload->fields, [&](const auto &item) { @@ -5881,9 +6121,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { const llvm::StringRef expected = expression.kind == "slot_get_valid" ? llvm::StringRef("i1") - : slot == plan.slots.end() - ? llvm::StringRef() - : llvm::StringRef(slot->payloadType); + : slot == plan.slots.end() ? llvm::StringRef() + : llvm::StringRef(slot->payloadType); if (!expression.operands.empty() || slot == plan.slots.end() || expression.type != expected) return planError("slot get expression type is inconsistent"); @@ -5907,8 +6146,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { if (aggregate == plan.aggregates.end() || aggregate->kind != "array" || aggregate->length != expression.selectionCount || aggregate->elements.size() != 1 || - aggregate->elements.front() != expression.type || - !elementWidth || expression.width != *elementWidth || !indexWidth || + aggregate->elements.front() != expression.type || !elementWidth || + expression.width != *elementWidth || !indexWidth || *indexWidth > 64 || expression.indexWidth != *indexWidth) return planError("dynamic value_array access types are inconsistent"); } else if (expression.kind == "array_update_dynamic") { @@ -5996,8 +6235,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { if (!width || *width > 64 || left == valueTypes.end() || right == valueTypes.end() || left->getValue() != expression.type || right->getValue() != expression.type) - return planError( - "bits arithmetic operands and result must share one i1..i64 type"); + return planError("bits arithmetic operands and result must share one " + "i1..i64 type"); } else if (expression.kind == "not") { auto operand = expression.operands.size() == 1 ? valueTypes.find(expression.operands.front()) @@ -6044,8 +6283,7 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { "comparisons before QueueGraph planning"); } else if (expression.kind == "udiv" || expression.kind == "urem") { if (expression.operands.size() != 2) - return planError( - "unsigned div/rem expression contract is malformed"); + return planError("unsigned div/rem expression contract is malformed"); auto left = valueTypes.find(expression.operands[0]); auto right = valueTypes.find(expression.operands[1]); auto resultWidth = bitsWidth(expression.type); @@ -6053,8 +6291,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { left->getValue() != expression.type || right->getValue() != expression.type || !resultWidth || !acir::isPrimitiveInputWidth(*resultWidth)) - return planError( - "unsigned div/rem operands and result must share one i1..i64 type"); + return planError("unsigned div/rem operands and result must share " + "one i1..i64 type"); } else if (expression.kind == "masked_match") { if (expression.operands.size() != 1 || expression.type != "i1") return planError("masked_match expression contract is malformed"); @@ -6068,6 +6306,99 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { auto value = parseExactWidthHex(expression.value, *inputWidth); if (!mask || !value || (*value & ~*mask) != 0) return planError("masked_match mask/value metadata is inconsistent"); + } else if (expression.kind == "divrem_quotient" || + expression.kind == "divrem_remainder") { + if (expression.operands.size() != 4 || expression.type != "i64") + return planError("divrem expression must have lhs, rhs, signed, and " + "word operands"); + for (size_t index = 0; index < expression.operands.size(); ++index) { + auto operand = valueTypes.find(expression.operands[index]); + const bool mode = index >= 2; + if (operand == valueTypes.end() || + operand->getValue() != (mode ? "i1" : "i64")) + return planError(mode ? "divrem mode operands must be i1" + : "divrem data operands must be i64"); + } + } else if (llvm::StringSwitch(expression.kind) + .Cases({"addw", "subw", "andw", "orw", "xorw", "sll", + "srl", "sra", "sllw", "srlw", "sraw", "smin", + "umin", "smax", "umax", "mulw"}, + true) + .Default(false)) { + if (expression.operands.size() != 2 || expression.type != "i64") + return planError("ALU binary expression must be two i64 operands"); + for (const std::string &operandName : expression.operands) { + auto operand = valueTypes.find(operandName); + if (operand == valueTypes.end() || operand->getValue() != "i64") + return planError("ALU binary operands must be i64"); + } + } else if (llvm::StringSwitch(expression.kind) + .Cases({"madd", "maddw", "msub"}, true) + .Default(false)) { + if (expression.operands.size() != 3 || expression.type != "i64") + return planError("ALU ternary expression must be three i64 operands"); + for (const std::string &operandName : expression.operands) { + auto operand = valueTypes.find(operandName); + if (operand == valueTypes.end() || operand->getValue() != "i64") + return planError("ALU ternary operands must be i64"); + } + } else if (expression.kind == "bitfield_extract" || + expression.kind == "bitfield_popcount" || + expression.kind == "bitfield_clz" || + expression.kind == "bitfield_ctz" || + expression.kind == "bitfield_clear" || + expression.kind == "bitfield_set" || + expression.kind == "bitfield_reverse_bytes") { + if (expression.operands.size() != 3 || expression.type != "i64") + return planError( + "ALU bitfield expression must have value, width, offset"); + auto value = valueTypes.find(expression.operands[0]); + auto width = valueTypes.find(expression.operands[1]); + auto offset = valueTypes.find(expression.operands[2]); + if (value == valueTypes.end() || value->getValue() != "i64" || + width == valueTypes.end() || width->getValue() != "i7" || + offset == valueTypes.end() || offset->getValue() != "i6") + return planError("ALU bitfield operands must be i64, i7, and i6"); + if (expression.kind == "bitfield_extract" && + expression.predicate != "signed" && + expression.predicate != "unsigned") + return planError("bitfield_extract signed mode is malformed"); + } else if (expression.kind == "bitfield_insert") { + if (expression.operands.size() != 4 || expression.type != "i64") + return planError( + "ALU bitfield insert must have value, source, width, offset"); + auto value = valueTypes.find(expression.operands[0]); + auto source = valueTypes.find(expression.operands[1]); + auto width = valueTypes.find(expression.operands[2]); + auto offset = valueTypes.find(expression.operands[3]); + if (value == valueTypes.end() || value->getValue() != "i64" || + source == valueTypes.end() || source->getValue() != "i64" || + width == valueTypes.end() || width->getValue() != "i7" || + offset == valueTypes.end() || offset->getValue() != "i6") + return planError( + "ALU bitfield insert operands must be i64, i64, i7, i6"); + } else if (expression.kind == "sext_low" || + expression.kind == "zext_low") { + if (expression.operands.size() != 2 || expression.type != "i64") + return planError("ALU extension must have value and width operands"); + auto value = valueTypes.find(expression.operands[0]); + auto width = valueTypes.find(expression.operands[1]); + if (value == valueTypes.end() || value->getValue() != "i64" || + width == valueTypes.end() || width->getValue() != "i7") + return planError("ALU extension operands must be i64 and i7"); + } else if (expression.kind == "csel") { + if (expression.operands.size() != 4 || expression.type != "i64") + return planError( + "ALU csel must have predicate, lhs, rhs, negate_false"); + auto predicate = valueTypes.find(expression.operands[0]); + auto lhs = valueTypes.find(expression.operands[1]); + auto rhs = valueTypes.find(expression.operands[2]); + auto negate = valueTypes.find(expression.operands[3]); + if (predicate == valueTypes.end() || predicate->getValue() != "i1" || + lhs == valueTypes.end() || lhs->getValue() != "i64" || + rhs == valueTypes.end() || rhs->getValue() != "i64" || + negate == valueTypes.end() || negate->getValue() != "i1") + return planError("ALU csel operands must be i1, i64, i64, i1"); } else if (expression.kind == "priority_index" || expression.kind == "priority_valid") { if (expression.operands.size() != 1 || @@ -6226,18 +6557,15 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { uint64_t upper = 0; bool valid = leftBounds && rightBounds && resultBounds; if (valid && expression.kind == "range_add") { - valid = rightBounds->first <= - std::numeric_limits::max() - - leftBounds->first && + valid = rightBounds->first <= std::numeric_limits::max() - + leftBounds->first && rightBounds->second <= - std::numeric_limits::max() - - leftBounds->second; + std::numeric_limits::max() - leftBounds->second; if (valid) { lower = leftBounds->first + rightBounds->first; upper = leftBounds->second + rightBounds->second; } - } - else if (valid) { + } else if (valid) { valid = leftBounds->first >= rightBounds->second; if (valid) { lower = leftBounds->first - rightBounds->second; @@ -6256,10 +6584,10 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { if (left == valueTypes.end() || right == valueTypes.end() || !rangeBounds(left->getValue()) || !rangeBounds(right->getValue()) || expression.type != "i1" || - !llvm::is_contained( - llvm::ArrayRef{"eq", "ne", "ult", "ule", - "ugt", "uge"}, - expression.predicate)) + !llvm::is_contained(llvm::ArrayRef{"eq", "ne", + "ult", "ule", + "ugt", "uge"}, + expression.predicate)) return planError("bounded comparison contract is malformed"); } else if (expression.kind == "range_wrap" || expression.kind == "range_saturate") { @@ -6354,9 +6682,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return planError("bit_insert expression contract is malformed"); auto base = valueTypes.find(expression.operands[0]); auto value = valueTypes.find(expression.operands[1]); - auto baseWidth = base == valueTypes.end() - ? std::optional() - : bitsWidth(base->getValue()); + auto baseWidth = base == valueTypes.end() ? std::optional() + : bitsWidth(base->getValue()); auto valueWidth = value == valueTypes.end() ? std::optional() : bitsWidth(value->getValue()); @@ -6366,7 +6693,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { *resultWidth != *baseWidth) return planError("bit_insert expression widths are inconsistent"); } else if (expression.kind == "snapshot_set") { - if (!expression.operands.empty() || expression.type != "state_reservation" || + if (!expression.operands.empty() || + expression.type != "state_reservation" || expression.field.empty() || !tables.contains(expression.table) || (expression.predicate != "complete" && expression.predicate != "fields")) @@ -6402,8 +6730,7 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { const auto *value = pair.value; const auto *valid = pair.valid; if (pair.valueCount != 1 || pair.validCount != 1 || !value || !valid || - value->field != valid->field || - value->operands != valid->operands || + value->field != valid->field || value->operands != valid->operands || value->staticTypeTarget != valid->staticTypeTarget) return planError( "checked range conversion requires one value/valid pair"); @@ -6463,8 +6790,7 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { ? constraints.find(expression.operands[1]) : constraints.end(); if (index == constraints.end() || expression.selectionCount == 0 || - !index->getValue().provesWithin( - 0, expression.selectionCount - 1)) + !index->getValue().provesWithin(0, expression.selectionCount - 1)) return planError("value_array index is not statically safe"); } if (expression.kind == "array_update_dynamic") { @@ -6472,8 +6798,7 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { ? constraints.find(expression.operands[1]) : constraints.end(); if (index == constraints.end() || expression.selectionCount == 0 || - !index->getValue().provesWithin( - 0, expression.selectionCount - 1)) + !index->getValue().provesWithin(0, expression.selectionCount - 1)) return planError("value_array update index is not statically safe"); } if (expression.kind == "range_refine") { @@ -6542,8 +6867,8 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { if (block.kind == "source" && llvm::any_of(block.outputs, [&](llvm::StringRef output) { llvm::StringSet<> active; - return containsDeclaredRange( - containsDeclaredRange, queueTypes.lookup(output), active); + return containsDeclaredRange(containsDeclaredRange, + queueTypes.lookup(output), active); })) return planError( "external source cannot carry an undecoded bounded range"); @@ -6588,15 +6913,13 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { auto predecessor = identities.find(block.yields[1]); auto resource = identities.find(block.yields[2]); auto cost = identities.find(block.yields[3]); - std::optional keyWidth = key == identities.end() - ? std::nullopt - : bitsWidth(key->getValue()); + std::optional keyWidth = + key == identities.end() ? std::nullopt : bitsWidth(key->getValue()); std::optional resourceWidth = resource == identities.end() ? std::nullopt : bitsWidth(resource->getValue()); - std::optional costWidth = cost == identities.end() - ? std::nullopt - : bitsWidth(cost->getValue()); + std::optional costWidth = + cost == identities.end() ? std::nullopt : bitsWidth(cost->getValue()); if (!keyWidth || *keyWidth == 0 || *keyWidth > 16 || predecessor == identities.end() || predecessor->getValue() != key->getValue() || !resourceWidth || @@ -6689,8 +7012,9 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return false; return llvm::StringSwitch(expression.kind) .Cases({"constant", "enum_constant", "get", "value_select"}, true) - .Cases({"add", "sub", "mul", "udiv", "urem", "and", "or", - "xor", "not"}, true) + .Cases({"add", "sub", "mul", "udiv", "sdiv", "urem", "srem", "and", + "or", "xor", "not"}, + true) .Cases({"shl", "shr", "extract", "insert", "concat"}, true) .Cases({"popcount", "count_zeros", "cmp", "masked_match"}, true) .Cases({"range_wrap", "range_saturate", "range_refine", @@ -6699,6 +7023,37 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { .Cases({"range_checked_value", "range_checked_valid", "array_get_dynamic", "array_update_dynamic"}, true) + .Cases({"addw", + "subw", + "andw", + "orw", + "xorw", + "sll", + "srl", + "sra", + "sllw", + "srlw", + "sraw", + "smin", + "umin", + "smax", + "umax", + "mulw", + "madd", + "maddw", + "msub", + "bitfield_extract", + "bitfield_popcount", + "bitfield_clz", + "bitfield_ctz", + "bitfield_clear", + "bitfield_set", + "bitfield_reverse_bytes", + "bitfield_insert", + "sext_low", + "zext_low", + "csel"}, + true) .Default(false); }; // Canonical identity is computed bottom-up over structurally pure @@ -6746,8 +7101,7 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { if (expression.kind == "mul" || expression.kind == "and" || expression.kind == "or" || expression.kind == "xor" || (expression.kind == "cmp" && - (expression.predicate == "eq" || - expression.predicate == "ne"))) + (expression.predicate == "eq" || expression.predicate == "ne"))) llvm::sort(operands); std::string key; llvm::raw_string_ostream stream(key); @@ -6787,56 +7141,55 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { return expression != block.expressions.end() && expression->kind == "constant" && expression->literal == "false"; }; - auto collectConjuncts = - [&](llvm::StringRef root, bool rootNegated, - llvm::DenseSet> &visited, - llvm::SmallVectorImpl &literals) { - // Same walk as the analysis-side collector: an explicit LIFO - // worklist keeps deep `and`/`mul` chains off the C++ stack, with - // operands pushed in reverse so literals keep their order. - llvm::SmallVector, 16> pending; - pending.push_back({root.str(), rootNegated}); - while (!pending.empty()) { - auto [identity, negated] = std::move(pending.back()); - pending.pop_back(); - const uint64_t atom = canonicalExpression(identity); - if (!visited.insert({atom, static_cast(negated)}) - .second) - continue; - size_t index = block.expressions.size(); - for (size_t candidate = 0; candidate < block.expressions.size(); - ++candidate) - if (block.expressions[candidate].result == identity) { - index = candidate; - break; - } - const bool found = index < block.expressions.size(); - if (found && !negated && block.expressions[index].type == "i1" && - (block.expressions[index].kind == "mul" || - block.expressions[index].kind == "and") && - block.expressions[index].operands.size() == 2) { - literals.push_back({atom, false}); - pending.push_back({block.expressions[index].operands[1], false}); - pending.push_back({block.expressions[index].operands[0], false}); - continue; - } - if (found && block.expressions[index].kind == "cmp" && - block.expressions[index].predicate == "eq" && - block.expressions[index].operands.size() == 2) { - if (isFalse(block.expressions[index].operands[0])) { - pending.push_back( - {block.expressions[index].operands[1], !negated}); - continue; - } - if (isFalse(block.expressions[index].operands[1])) { - pending.push_back( - {block.expressions[index].operands[0], !negated}); - continue; - } - } - literals.push_back({atom, negated}); + auto collectConjuncts = [&](llvm::StringRef root, bool rootNegated, + llvm::DenseSet> + &visited, + llvm::SmallVectorImpl &literals) { + // Same walk as the analysis-side collector: an explicit LIFO + // worklist keeps deep `and`/`mul` chains off the C++ stack, with + // operands pushed in reverse so literals keep their order. + llvm::SmallVector, 16> pending; + pending.push_back({root.str(), rootNegated}); + while (!pending.empty()) { + auto [identity, negated] = std::move(pending.back()); + pending.pop_back(); + const uint64_t atom = canonicalExpression(identity); + if (!visited.insert({atom, static_cast(negated)}).second) + continue; + size_t index = block.expressions.size(); + for (size_t candidate = 0; candidate < block.expressions.size(); + ++candidate) + if (block.expressions[candidate].result == identity) { + index = candidate; + break; } - }; + const bool found = index < block.expressions.size(); + if (found && !negated && block.expressions[index].type == "i1" && + (block.expressions[index].kind == "mul" || + block.expressions[index].kind == "and") && + block.expressions[index].operands.size() == 2) { + literals.push_back({atom, false}); + pending.push_back({block.expressions[index].operands[1], false}); + pending.push_back({block.expressions[index].operands[0], false}); + continue; + } + if (found && block.expressions[index].kind == "cmp" && + block.expressions[index].predicate == "eq" && + block.expressions[index].operands.size() == 2) { + if (isFalse(block.expressions[index].operands[0])) { + pending.push_back( + {block.expressions[index].operands[1], !negated}); + continue; + } + if (isFalse(block.expressions[index].operands[1])) { + pending.push_back( + {block.expressions[index].operands[0], !negated}); + continue; + } + } + literals.push_back({atom, negated}); + } + }; auto mutuallyExclusive = [&](llvm::StringRef left, llvm::StringRef right) { llvm::SmallVector leftLiterals; @@ -6868,11 +7221,10 @@ llvm::Error verifyQueueGraphPlan(const QueueGraphPlan &plan) { const bool disjointFields = writes[left]->mode == "field" && writes[right]->mode == "field" && - llvm::none_of(writes[left]->fields, - [&](const std::string &field) { - return llvm::is_contained( - writes[right]->fields, field); - }); + llvm::none_of( + writes[left]->fields, [&](const std::string &field) { + return llvm::is_contained(writes[right]->fields, field); + }); if (!disjoint && !exclusive && !disjointFields) return planError( "same-owner firing writes have an unresolved index/field " @@ -7080,8 +7432,7 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { value["symbol"] = frame.symbol; frames.push_back(std::move(value)); } - origins.push_back( - llvm::json::Object{{"frames", std::move(frames)}}); + origins.push_back(llvm::json::Object{{"frames", std::move(frames)}}); } return llvm::json::Object{{"origins", std::move(origins)}}; }; @@ -7184,8 +7535,7 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { result["initial_cursor"] = expression.initialCursor; } if (!expression.sourceProvenance.origins.empty()) - result["source_provenance"] = - provenanceJson(expression.sourceProvenance); + result["source_provenance"] = provenanceJson(expression.sourceProvenance); return result; }; auto initValueJson = @@ -7276,14 +7626,11 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { llvm::json::Array laneOrdinals; for (uint64_t lane : queue.laneOrdinals) laneOrdinals.push_back(lane); - llvm::json::Object value{{"depth", queue.depth}, - {"lane_ordinals", std::move(laneOrdinals)}, - {"lanes", queue.lanes}, - {"latency", queue.latency}, - {"name", queue.name}, - {"payload_type", queue.payloadType}, - {"rate", queue.rate}, - {"scope", queue.scope}}; + llvm::json::Object value{ + {"depth", queue.depth}, {"lane_ordinals", std::move(laneOrdinals)}, + {"lanes", queue.lanes}, {"latency", queue.latency}, + {"name", queue.name}, {"payload_type", queue.payloadType}, + {"rate", queue.rate}, {"scope", queue.scope}}; if (queue.payloadProjection) { llvm::json::Array fields; for (const std::string &field : queue.payloadProjection->keptFields) @@ -7371,8 +7718,8 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { } llvm::json::Array slotReleases; for (const SlotReleaseEffectPlan &release : block.slotReleases) - slotReleases.push_back(llvm::json::Object{{"slot", release.slot}, - {"when", release.when}}); + slotReleases.push_back( + llvm::json::Object{{"slot", release.slot}, {"when", release.when}}); llvm::json::Array outputPresence; for (const OutputPresencePlan &output : block.outputPresence) outputPresence.push_back(llvm::json::Object{{"ordinal", output.ordinal}, @@ -7429,22 +7776,18 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { {"write_fields", std::move(writeFields)}, {"yields", std::move(yields)}}; if (!block.sourceProvenance.origins.empty()) - blockValue["source_provenance"] = - provenanceJson(block.sourceProvenance); + blockValue["source_provenance"] = provenanceJson(block.sourceProvenance); blockValues.push_back(std::move(blockValue)); } llvm::json::Array memoryInstanceValues; for (const MemoryInstancePlan &instance : memoryInstances) { - llvm::json::Object value{{"data_type", instance.dataType}, - {"entries", instance.entries}, - {"init", instance.init}, - {"latency", instance.latency}, - {"name", instance.name}, - {"owner_path", instance.ownerPath}, - {"stable_id", instance.stableId}}; + llvm::json::Object value{ + {"data_type", instance.dataType}, {"entries", instance.entries}, + {"init", instance.init}, {"latency", instance.latency}, + {"name", instance.name}, {"owner_path", instance.ownerPath}, + {"stable_id", instance.stableId}}; if (!instance.sourceProvenance.origins.empty()) - value["source_provenance"] = - provenanceJson(instance.sourceProvenance); + value["source_provenance"] = provenanceJson(instance.sourceProvenance); memoryInstanceValues.push_back(std::move(value)); } llvm::json::Array memoryRequestValues; @@ -7470,19 +7813,19 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { for (const TableInitValuePlan &value : table.initImage) initImage.push_back(initValueJson(initValueJson, value)); llvm::json::Object value{{"axis_widths", std::move(axisWidths)}, - {"entries", table.entries}, - {"entry_type", table.entryType}, - {"init", table.init}, - {"init_image", std::move(initImage)}, - {"init_version", table.initVersion}, - {"has_typed_schema", table.hasTypedSchema}, - {"layout", table.layout}, - {"layout_version", table.layoutVersion}, - {"name", table.name}, - {"owner_path", table.ownerPath}, - {"schema_id", table.schemaId}, - {"shape", std::move(shape)}, - {"stable_id", table.stableId}}; + {"entries", table.entries}, + {"entry_type", table.entryType}, + {"init", table.init}, + {"init_image", std::move(initImage)}, + {"init_version", table.initVersion}, + {"has_typed_schema", table.hasTypedSchema}, + {"layout", table.layout}, + {"layout_version", table.layoutVersion}, + {"name", table.name}, + {"owner_path", table.ownerPath}, + {"schema_id", table.schemaId}, + {"shape", std::move(shape)}, + {"stable_id", table.stableId}}; if (!table.sourceProvenance.origins.empty()) value["source_provenance"] = provenanceJson(table.sourceProvenance); tableValues.push_back(std::move(value)); @@ -7520,8 +7863,7 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { matchValue["scan_bound"] = scanBound; } if (!match.sourceProvenance.origins.empty()) - matchValue["source_provenance"] = - provenanceJson(match.sourceProvenance); + matchValue["source_provenance"] = provenanceJson(match.sourceProvenance); tableMatchValues.push_back(std::move(matchValue)); } llvm::json::Array tableSelectionValues; @@ -7583,12 +7925,10 @@ llvm::Expected QueueGraphPlan::canonicalJson() const { } llvm::json::Array slotValues; for (const SlotPlan &slot : slots) { - llvm::json::Object value{{"input", slot.input}, - {"name", slot.name}, - {"owner_path", slot.ownerPath}, - {"payload_type", slot.payloadType}, - {"scope", slot.scope}, - {"stable_id", slot.stableId}}; + llvm::json::Object value{ + {"input", slot.input}, {"name", slot.name}, + {"owner_path", slot.ownerPath}, {"payload_type", slot.payloadType}, + {"scope", slot.scope}, {"stable_id", slot.stableId}}; if (!slot.sourceProvenance.origins.empty()) value["source_provenance"] = provenanceJson(slot.sourceProvenance); slotValues.push_back(std::move(value)); @@ -7766,14 +8106,13 @@ llvm::Expected QueueGraphPlan::sourceMapJson() const { value["symbol"] = frame.symbol; frames.push_back(std::move(value)); } - origins.push_back( - llvm::json::Object{{"frames", std::move(frames)}}); + origins.push_back(llvm::json::Object{{"frames", std::move(frames)}}); } return llvm::json::Object{{"origins", std::move(origins)}}; }; - auto expressionJson = [&](auto &&self, - const QueueExpressionPlan &expression) - -> llvm::json::Object { + auto expressionJson = + [&](auto &&self, + const QueueExpressionPlan &expression) -> llvm::json::Object { llvm::json::Array nested; for (const QueueExpressionPlan &child : expression.nestedExpressions) nested.push_back(self(self, child)); @@ -7879,15 +8218,14 @@ llvm::Expected QueueGraphPlan::sourceMapJson() const { {"blocks", std::move(blockValues)}, {"contract_epoch", "0.5"}, {"definition", definition.empty() ? llvm::json::Value(nullptr) - : llvm::json::Value(definition)}, + : llvm::json::Value(definition)}, {"helpers", std::move(helperValues)}, {"module_instances", std::move(instanceValues)}, {"module_specializations", std::move(specializationValues)}, {"schema", "agentic-circuit-source-map"}, - {"specialization", - specializationFingerprint.empty() - ? llvm::json::Value(nullptr) - : llvm::json::Value(specializationFingerprint)}, + {"specialization", specializationFingerprint.empty() + ? llvm::json::Value(nullptr) + : llvm::json::Value(specializationFingerprint)}, {"state_owners", std::move(stateOwnerValues)}, {"system", system}, {"table_matches", std::move(tableMatchValues)}, diff --git a/compiler/acir/lib/CodeGen/QueueGraphPyc.cpp b/compiler/acir/lib/CodeGen/QueueGraphPyc.cpp index 88ca17851..e7bd9eba7 100644 --- a/compiler/acir/lib/CodeGen/QueueGraphPyc.cpp +++ b/compiler/acir/lib/CodeGen/QueueGraphPyc.cpp @@ -4,8 +4,8 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallString.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" @@ -67,8 +67,7 @@ inlineSourceProvenance(const QueueSourceProvenancePlan &definition, return result; } -std::string sourceLocationSuffix( - const QueueSourceProvenancePlan &provenance) { +std::string sourceLocationSuffix(const QueueSourceProvenancePlan &provenance) { std::vector origins; for (const QueueSourceOriginPlan &origin : provenance.origins) { if (origin.empty()) @@ -97,9 +96,9 @@ std::string sourceLocationSuffix( return result; } -std::string attachPycSourceLocations( - llvm::StringRef body, - const llvm::StringMap &sourceLocations) { +std::string +attachPycSourceLocations(llvm::StringRef body, + const llvm::StringMap &sourceLocations) { std::string result; llvm::SmallVector lines; body.split(lines, '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/true); @@ -109,9 +108,8 @@ std::string attachPycSourceLocations( if (trimmed.consume_front("// pyc-source")) { currentLocation = trimmed.str(); } else if (trimmed.starts_with('%')) { - llvm::StringRef value = trimmed.take_until([](char character) { - return character == ' ' || character == '='; - }); + llvm::StringRef value = trimmed.take_until( + [](char character) { return character == ' ' || character == '='; }); auto found = sourceLocations.find(value); if (found != sourceLocations.end() && !line.contains(" loc(")) result.append(line).append(found->getValue()); @@ -181,8 +179,7 @@ const QueueHelperPlan *findHelper(const QueueGraphPlan &plan, return found == plan.helpers.end() ? nullptr : &*found; } -std::optional> -rangeBounds(llvm::StringRef type) { +std::optional> rangeBounds(llvm::StringRef type) { constexpr llvm::StringLiteral prefix = "!ac.range<"; if (!type.starts_with(prefix) || !type.ends_with('>')) return std::nullopt; @@ -203,9 +200,9 @@ llvm::Expected typeWidth(const QueueGraphPlan &plan, return width; } if (auto bounds = rangeBounds(type)) - return bounds->second == std::numeric_limits::max() - ? 64 - : std::max(1u, llvm::Log2_64_Ceil(bounds->second + 1)); + return bounds->second == std::numeric_limits::max() + ? 64 + : std::max(1u, llvm::Log2_64_Ceil(bounds->second + 1)); if (const QueueEnumPlan *enumeration = findEnum(plan, type)) if (enumeration->width <= kMaximumPackedValueWidth) return static_cast(enumeration->width); @@ -713,8 +710,8 @@ generateLaneQueuePyc(const QueueGraphPlan &plan, return sourceMap.takeError(); output << "module attributes {pyc.top = @" << plan.system << ", pyc.frontend.contract = \"pycircuit\", pyc.source_map = " - << mlirStringLiteral(*sourceMap) << "} {\n func.func @" - << plan.system << '('; + << mlirStringLiteral(*sourceMap) << "} {\n func.func @" << plan.system + << '('; writeList(output, arguments); output << ") -> ("; writeList(output, resultTypes); @@ -761,6 +758,7 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, for (const auto &entry : *inheritedTypes) types[entry.getKey()] = entry.getValue(); llvm::StringMap> priorityValues; + llvm::StringMap> divremResults; struct ChoiceValues { std::vector indices; std::vector valids; @@ -806,9 +804,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, llvm::StringRef resultType = {}) { std::string result = newValue(); const bool compare = operation == "ult" || operation == "eq"; - body << " " << result << " = pyc." - << (compare ? "cmp" : operation.str()) << ' ' << lhs.str() << ", " - << rhs.str(); + body << " " << result << " = pyc." << (compare ? "cmp" : operation.str()) + << ' ' << lhs.str() << ", " << rhs.str(); if (compare) body << " {predicate = \"" << operation.str() << "\"}"; body << " : " << type.str() << ", " << type.str() << " -> " @@ -834,7 +831,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, unsigned targetWidth) { if (sourceWidth == targetWidth) return input.str(); - std::string zero = emitPycConstant(0, "i" + std::to_string(targetWidth - sourceWidth)); + std::string zero = + emitPycConstant(0, "i" + std::to_string(targetWidth - sourceWidth)); std::string result = newValue(); body << " " << result << " = pyc.concat(" << zero << ", " << input.str() << ") : (i" << targetWidth - sourceWidth << ", i" << sourceWidth @@ -938,10 +936,10 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, : llvm::StringRef(expression.type); auto bounds = rangeBounds(targetType); auto sourceLogicalType = valueType(expression.operands.front()); - auto sourceWidth = sourceLogicalType - ? typeWidth(plan, *sourceLogicalType) - : llvm::Expected( - sourceLogicalType.takeError()); + auto sourceWidth = + sourceLogicalType + ? typeWidth(plan, *sourceLogicalType) + : llvm::Expected(sourceLogicalType.takeError()); auto targetWidth = typeWidth(plan, targetType); if (!bounds || !sourceWidth) return !sourceWidth ? sourceWidth.takeError() @@ -961,10 +959,9 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, }; if (expression.kind == "range_wrap") { std::string normalized; - const uint64_t targetMask = - *targetWidth == 64 - ? std::numeric_limits::max() - : (uint64_t{1} << *targetWidth) - 1; + const uint64_t targetMask = *targetWidth == 64 + ? std::numeric_limits::max() + : (uint64_t{1} << *targetWidth) - 1; if (bounds->first == 0 && bounds->second == targetMask) { normalized = raw; } else { @@ -991,10 +988,9 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, } else { auto cached = checkedRangeValues.find(expression.literal); if (cached == checkedRangeValues.end()) { - const std::string valid = emitPycBinary( - "and", emitPycNot(below), emitPycNot(above), "i1"); - const std::string checked = - emitPycSelect(valid, raw, lower, type); + const std::string valid = emitPycBinary("and", emitPycNot(below), + emitPycNot(above), "i1"); + const std::string checked = emitPycSelect(valid, raw, lower, type); checkedRangeValues[expression.literal] = { narrowUnsigned(checked, width, *targetWidth), valid}; cached = checkedRangeValues.find(expression.literal); @@ -1005,18 +1001,18 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, } } else if (expression.kind == "range_refine") { auto sourceLogicalType = valueType(expression.operands.front()); - auto sourceWidth = sourceLogicalType - ? typeWidth(plan, *sourceLogicalType) - : llvm::Expected( - sourceLogicalType.takeError()); + auto sourceWidth = + sourceLogicalType + ? typeWidth(plan, *sourceLogicalType) + : llvm::Expected(sourceLogicalType.takeError()); auto targetWidth = typeWidth(plan, expression.type); if (!sourceWidth) return sourceWidth.takeError(); if (!targetWidth) return targetWidth.takeError(); const unsigned width = std::max(*sourceWidth, *targetWidth); - result = narrowUnsigned( - widenUnsigned(*first, *sourceWidth, width), width, *targetWidth); + result = narrowUnsigned(widenUnsigned(*first, *sourceWidth, width), + width, *targetWidth); } else if (expression.kind == "range_bits") { result = *first; } else if (expression.kind == "range_add" || @@ -1024,14 +1020,14 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, auto right = value(expression.operands[1]); auto leftLogicalType = valueType(expression.operands[0]); auto rightLogicalType = valueType(expression.operands[1]); - auto leftWidth = leftLogicalType - ? typeWidth(plan, *leftLogicalType) - : llvm::Expected( - leftLogicalType.takeError()); - auto rightWidth = rightLogicalType - ? typeWidth(plan, *rightLogicalType) - : llvm::Expected( - rightLogicalType.takeError()); + auto leftWidth = + leftLogicalType + ? typeWidth(plan, *leftLogicalType) + : llvm::Expected(leftLogicalType.takeError()); + auto rightWidth = + rightLogicalType + ? typeWidth(plan, *rightLogicalType) + : llvm::Expected(rightLogicalType.takeError()); auto resultWidth = typeWidth(plan, expression.type); if (!right) return right.takeError(); @@ -1044,23 +1040,23 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, const unsigned width = std::max({*leftWidth, *rightWidth, *resultWidth}); const std::string type = "i" + std::to_string(width); - std::string computed = emitPycBinary( - expression.kind == "range_add" ? "add" : "sub", - widenUnsigned(*first, *leftWidth, width), - widenUnsigned(*right, *rightWidth, width), type); + std::string computed = + emitPycBinary(expression.kind == "range_add" ? "add" : "sub", + widenUnsigned(*first, *leftWidth, width), + widenUnsigned(*right, *rightWidth, width), type); result = narrowUnsigned(computed, width, *resultWidth); } else if (expression.kind == "range_cmp") { auto right = value(expression.operands[1]); auto leftLogicalType = valueType(expression.operands[0]); auto rightLogicalType = valueType(expression.operands[1]); - auto leftWidth = leftLogicalType - ? typeWidth(plan, *leftLogicalType) - : llvm::Expected( - leftLogicalType.takeError()); - auto rightWidth = rightLogicalType - ? typeWidth(plan, *rightLogicalType) - : llvm::Expected( - rightLogicalType.takeError()); + auto leftWidth = + leftLogicalType + ? typeWidth(plan, *leftLogicalType) + : llvm::Expected(leftLogicalType.takeError()); + auto rightWidth = + rightLogicalType + ? typeWidth(plan, *rightLogicalType) + : llvm::Expected(rightLogicalType.takeError()); if (!right) return right.takeError(); if (!leftWidth) @@ -1091,7 +1087,7 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, if (negate) result = emitPycNot(result); } else if (expression.kind == "table_selection_index_ref" || - expression.kind == "table_selection_valid_ref") { + expression.kind == "table_selection_valid_ref") { const std::string sharedKey = "selection:" + expression.table + ":" + expression.field + ":" + expression.kind + ":" + std::to_string(expression.laneOrdinal); @@ -1158,11 +1154,10 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, evaluation.expressions.push_back(std::move(choice)); } llvm::StringMap emitted; - auto selected = - emitTransform(plan, evaluation, {}, {}, 0, nextValue, body, - &emitted, tableValues, roundRobinStates, - choiceOwner, &values, &types, sharedTableValues, - sourceLocations, inlineCallsite); + auto selected = emitTransform( + plan, evaluation, {}, {}, 0, nextValue, body, &emitted, + tableValues, roundRobinStates, choiceOwner, &values, &types, + sharedTableValues, sourceLocations, inlineCallsite); if (!selected) return selected.takeError(); for (const QueueExpressionPlan &reference : block.expressions) { @@ -1221,10 +1216,10 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, QueueBlockPlan matchBlock; matchBlock.expressions.push_back(std::move(materialized)); matchBlock.yields.push_back(expression.result); - auto emitted = emitTransform(plan, matchBlock, {}, {}, 0, nextValue, - body, nullptr, tableValues, nullptr, {}, - nullptr, nullptr, nullptr, - sourceLocations, inlineCallsite); + auto emitted = + emitTransform(plan, matchBlock, {}, {}, 0, nextValue, body, + nullptr, tableValues, nullptr, {}, nullptr, nullptr, + nullptr, sourceLocations, inlineCallsite); if (!emitted) return emitted.takeError(); result = std::move(*emitted); @@ -1349,12 +1344,11 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, return pyc.takeError(); selectionKeyType = std::move(*pyc); for (uint64_t tableIndex : tableIndices) { - auto key = - emitTransform(plan, keyBlock, {table->getValue()[tableIndex]}, - {tablePlan->entryType}, 0, nextValue, body, - nullptr, tableValues, nullptr, {}, nullptr, - nullptr, nullptr, sourceLocations, - inlineCallsite); + auto key = emitTransform( + plan, keyBlock, {table->getValue()[tableIndex]}, + {tablePlan->entryType}, 0, nextValue, body, nullptr, + tableValues, nullptr, {}, nullptr, nullptr, nullptr, + sourceLocations, inlineCallsite); if (!key) return key.takeError(); candidateKeys.push_back(std::move(*key)); @@ -1951,6 +1945,33 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, body << " " << result << " = pyc.count_zeros " << *first << " {direction = \"" << expression.predicate << "\"} : " << *sourceType << " -> " << *resultType << "\n"; + } else if (expression.kind == "divrem_quotient" || + expression.kind == "divrem_remainder") { + if (expression.operands.size() != 4 || expression.type != "i64") + return pycError("divrem expression arity or result type mismatch"); + auto second = value(expression.operands[1]); + auto signedMode = value(expression.operands[2]); + auto wordMode = value(expression.operands[3]); + if (!second || !signedMode || !wordMode) + return pycError("divrem expression references unknown value"); + const std::string key = + expression.operands[0] + ":" + expression.operands[1] + ":" + + expression.operands[2] + ":" + expression.operands[3]; + auto found = divremResults.find(key); + if (found == divremResults.end()) { + std::string quotient = newValue(); + std::string remainder = newValue(); + body << " " << quotient << ", " << remainder << " = pyc.divrem " + << *first << ", " << *second << ", " << *signedMode << ", " + << *wordMode << " : (i64, i64, i1, i1) -> (i64, i64)\n"; + found = divremResults.try_emplace(key, quotient, remainder).first; + } + result = expression.kind == "divrem_quotient" + ? found->getValue().first + : found->getValue().second; + values[expression.result] = result; + types[expression.result] = expression.type; + continue; } else if (expression.kind == "array_get_dynamic") { if (expression.operands.size() != 2 || expression.width == 0 || expression.selectionCount == 0) @@ -1963,18 +1984,18 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, return aggregate.takeError(); if (!index) return index.takeError(); - auto aggregateType = aggregateLogicalType - ? pycType(plan, *aggregateLogicalType) - : llvm::Expected( - aggregateLogicalType.takeError()); - auto indexType = indexLogicalType - ? pycType(plan, *indexLogicalType) - : llvm::Expected( - indexLogicalType.takeError()); - auto indexWidth = indexLogicalType - ? typeWidth(plan, *indexLogicalType) - : llvm::Expected( - indexLogicalType.takeError()); + auto aggregateType = + aggregateLogicalType + ? pycType(plan, *aggregateLogicalType) + : llvm::Expected(aggregateLogicalType.takeError()); + auto indexType = + indexLogicalType + ? pycType(plan, *indexLogicalType) + : llvm::Expected(indexLogicalType.takeError()); + auto indexWidth = + indexLogicalType + ? typeWidth(plan, *indexLogicalType) + : llvm::Expected(indexLogicalType.takeError()); auto resultType = pycType(plan, expression.type); if (!aggregateType) return aggregateType.takeError(); @@ -1985,9 +2006,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, if (!resultType) return resultType.takeError(); const unsigned comparisonWidth = std::max( - *indexWidth, - std::max( - 1, llvm::Log2_64_Ceil(expression.selectionCount))); + *indexWidth, std::max( + 1, llvm::Log2_64_Ceil(expression.selectionCount))); const std::string comparisonType = "i" + std::to_string(comparisonWidth); const std::string widenedIndex = @@ -2014,12 +2034,12 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, next.push_back(std::move(candidates[index])); continue; } - std::string selected = emitPycSelect( - candidates[index].first, candidates[index].second, - candidates[index + 1].second, *resultType); - std::string valid = emitPycBinary( - "or", candidates[index].first, candidates[index + 1].first, - "i1"); + std::string selected = + emitPycSelect(candidates[index].first, candidates[index].second, + candidates[index + 1].second, *resultType); + std::string valid = + emitPycBinary("or", candidates[index].first, + candidates[index + 1].first, "i1"); next.emplace_back(std::move(valid), std::move(selected)); } candidates = std::move(next); @@ -2040,19 +2060,19 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, return index.takeError(); if (!replacement) return replacement.takeError(); - auto aggregateType = aggregateLogicalType - ? pycType(plan, *aggregateLogicalType) - : llvm::Expected( - aggregateLogicalType.takeError()); - auto indexWidth = indexLogicalType - ? typeWidth(plan, *indexLogicalType) - : llvm::Expected( - indexLogicalType.takeError()); + auto aggregateType = + aggregateLogicalType + ? pycType(plan, *aggregateLogicalType) + : llvm::Expected(aggregateLogicalType.takeError()); + auto indexWidth = + indexLogicalType + ? typeWidth(plan, *indexLogicalType) + : llvm::Expected(indexLogicalType.takeError()); auto elementLogicalType = valueType(expression.operands[2]); - auto elementType = elementLogicalType - ? pycType(plan, *elementLogicalType) - : llvm::Expected( - elementLogicalType.takeError()); + auto elementType = + elementLogicalType + ? pycType(plan, *elementLogicalType) + : llvm::Expected(elementLogicalType.takeError()); if (!aggregateType) return aggregateType.takeError(); if (!indexWidth) @@ -2060,9 +2080,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, if (!elementType) return elementType.takeError(); const unsigned comparisonWidth = std::max( - *indexWidth, - std::max( - 1, llvm::Log2_64_Ceil(expression.selectionCount))); + *indexWidth, std::max( + 1, llvm::Log2_64_Ceil(expression.selectionCount))); const std::string comparisonType = "i" + std::to_string(comparisonWidth); const std::string widenedIndex = @@ -2079,8 +2098,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, std::string ordinal = emitPycConstant(element, comparisonType); std::string selected = emitPycBinary("eq", widenedIndex, ordinal, comparisonType); - elements.push_back(emitPycSelect( - selected, *replacement, oldValue, *elementType)); + elements.push_back( + emitPycSelect(selected, *replacement, oldValue, *elementType)); } result = newValue(); body << " " << result << " = pyc.concat("; @@ -2096,6 +2115,220 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, body << *elementType; } body << ") -> " << *aggregateType << "\n"; + } else if (expression.kind == "addw" || expression.kind == "subw" || + expression.kind == "andw" || expression.kind == "orw" || + expression.kind == "xorw" || expression.kind == "sll" || + expression.kind == "srl" || expression.kind == "sra" || + expression.kind == "sllw" || expression.kind == "srlw" || + expression.kind == "sraw" || expression.kind == "smin" || + expression.kind == "umin" || expression.kind == "smax" || + expression.kind == "umax" || expression.kind == "mulw" || + expression.kind == "madd" || expression.kind == "maddw" || + expression.kind == "msub" || expression.kind == "csel") { + auto emitBinary = [&](llvm::StringRef operation, llvm::StringRef lhs, + llvm::StringRef rhs, llvm::StringRef type) { + std::string lowered = newValue(); + body << " " << lowered << " = pyc." << operation.str() << ' ' + << lhs.str() << ", " << rhs.str() << " : " << type.str() << ", " + << type.str() << " -> " << type.str() << "\n"; + return lowered; + }; + auto emitExtractLow = [&](llvm::StringRef input, + llvm::StringRef sourceType, + llvm::StringRef resultType) { + std::string lowered = newValue(); + body << " " << lowered << " = pyc.extract " << input.str() + << " {lsb = 0} : " << sourceType.str() << " -> " + << resultType.str() << "\n"; + return lowered; + }; + auto emitCast = [&](llvm::StringRef operation, llvm::StringRef input, + llvm::StringRef sourceType, + llvm::StringRef resultType) { + std::string lowered = newValue(); + body << " " << lowered << " = pyc." << operation.str() << ' ' + << input.str() << " : " << sourceType.str() << " -> " + << resultType.str() << "\n"; + return lowered; + }; + auto emitShift = [&](llvm::StringRef operation, llvm::StringRef lhs, + llvm::StringRef amount, llvm::StringRef type) { + std::string lowered = newValue(); + body << " " << lowered << " = pyc." << operation.str() << ' ' + << lhs.str() << ", " << amount.str() << " : " << type.str() + << ", " << type.str() << "\n"; + return lowered; + }; + if (expression.operands.size() < 2 || expression.type != "i64") + return pycError( + "ALU expression has malformed operands or result type"); + auto second = value(expression.operands[1]); + if (!second) + return second.takeError(); + auto emitWordBinary = [&](llvm::StringRef operation) { + std::string lhs = emitExtractLow(*first, "i64", "i32"); + std::string rhs = emitExtractLow(*second, "i64", "i32"); + std::string low = emitBinary(operation, lhs, rhs, "i32"); + return emitCast("sext", low, "i32", "i64"); + }; + if (expression.kind == "addw") + result = emitWordBinary("add"); + else if (expression.kind == "subw") + result = emitWordBinary("sub"); + else if (expression.kind == "andw") + result = emitWordBinary("and"); + else if (expression.kind == "orw") + result = emitWordBinary("or"); + else if (expression.kind == "xorw") + result = emitWordBinary("xor"); + else if (expression.kind == "mulw") + result = emitWordBinary("mul"); + else if (expression.kind == "sll" || expression.kind == "srl" || + expression.kind == "sra") { + std::string amount = + emitBinary("and", *second, emitPycConstant(63, "i64"), "i64"); + result = emitShift(expression.kind == "sll" + ? "shl" + : expression.kind == "srl" ? "lshr" : "ashr", + *first, amount, "i64"); + } else if (expression.kind == "sllw" || expression.kind == "srlw" || + expression.kind == "sraw") { + std::string lhs = emitExtractLow(*first, "i64", "i32"); + std::string amount64 = + emitBinary("and", *second, emitPycConstant(31, "i64"), "i64"); + std::string amount = emitCast("trunc", amount64, "i64", "i32"); + std::string low = + emitShift(expression.kind == "sllw" + ? "shl" + : expression.kind == "srlw" ? "lshr" : "ashr", + lhs, amount, "i32"); + result = emitCast("sext", low, "i32", "i64"); + } else if (expression.kind == "smin" || expression.kind == "smax" || + expression.kind == "umin" || expression.kind == "umax") { + std::string comparison = newValue(); + body << " " << comparison << " = pyc.cmp " << *first << ", " + << *second << " {predicate = \"" + << (expression.kind.front() == 's' ? "slt" : "ult") + << "\"} : i64, i64 -> i1\n"; + if (expression.kind.ends_with("max")) + comparison = emitPycNot(comparison); + result = emitPycSelect(comparison, *first, *second, "i64"); + } else if (expression.kind == "madd" || expression.kind == "maddw" || + expression.kind == "msub") { + if (expression.operands.size() != 3) + return pycError("ALU ternary expression arity mismatch"); + auto third = value(expression.operands[2]); + if (!third) + return third.takeError(); + if (expression.kind == "maddw") { + std::string lhs = emitExtractLow(*first, "i64", "i32"); + std::string rhs = emitExtractLow(*second, "i64", "i32"); + std::string addend = emitExtractLow(*third, "i64", "i32"); + std::string product = emitBinary("mul", lhs, rhs, "i32"); + result = emitCast("sext", emitBinary("add", product, addend, "i32"), + "i32", "i64"); + } else { + std::string product = emitBinary("mul", *first, *second, "i64"); + result = expression.kind == "madd" + ? emitBinary("add", product, *third, "i64") + : emitBinary("sub", *third, product, "i64"); + } + } else { + if (expression.operands.size() != 4) + return pycError("ALU select expression arity mismatch"); + auto lhs = value(expression.operands[1]); + auto rhs = value(expression.operands[2]); + auto negate = value(expression.operands[3]); + if (!lhs || !rhs || !negate) + return pycError("ALU select expression references unknown value"); + std::string negated = + emitBinary("sub", emitPycConstant(0, "i64"), *rhs, "i64"); + std::string choice = emitPycSelect(*negate, negated, *rhs, "i64"); + result = emitPycSelect(*first, *lhs, choice, "i64"); + } + } else if (expression.kind == "bitfield_extract" || + expression.kind == "bitfield_popcount" || + expression.kind == "bitfield_clz" || + expression.kind == "bitfield_ctz" || + expression.kind == "bitfield_clear" || + expression.kind == "bitfield_set" || + expression.kind == "bitfield_reverse_bytes" || + expression.kind == "bitfield_insert" || + expression.kind == "sext_low" || + expression.kind == "zext_low") { + auto emitCast = [&](llvm::StringRef operation, llvm::StringRef input, + llvm::StringRef sourceType, + llvm::StringRef resultType) { + std::string lowered = newValue(); + body << " " << lowered << " = pyc." << operation.str() << ' ' + << input.str() << " : " << sourceType.str() << " -> " + << resultType.str() << "\n"; + return lowered; + }; + auto emitOffset = [&](llvm::StringRef input) { + return emitCast("zext", input, "i6", "i7"); + }; + auto emitWrappingBitfield = [&](llvm::StringRef source, + llvm::StringRef width, + llvm::StringRef offset, uint64_t mode) { + std::string modeValue = emitPycConstant(mode, "i4"); + std::string lowered = newValue(); + body << " " << lowered << " = pyc.wrapping_bitfield " << *first + << ", " << source.str() << ", " << width.str() << ", " + << offset.str() << ", " << modeValue + << " {semantic_id = \"pyc.wrapping_bitfield.v1\"} : " + "(i64, i64, i7, i7, i4) -> i64\n"; + return lowered; + }; + if (expression.kind == "sext_low" || expression.kind == "zext_low") { + if (expression.operands.size() != 2) + return pycError("ALU extension expression arity mismatch"); + auto width = value(expression.operands[1]); + if (!width) + return width.takeError(); + result = emitWrappingBitfield(emitPycConstant(0, "i64"), *width, + emitPycConstant(0, "i7"), + expression.kind == "sext_low" ? 5 : 0); + } else { + if (expression.operands.size() < 3) + return pycError("ALU bitfield expression arity mismatch"); + auto width = value(expression.operands[1]); + auto offsetRaw = value(expression.operands[2]); + if (!width || !offsetRaw) + return pycError("ALU bitfield expression references unknown value"); + std::string source = emitPycConstant(0, "i64"); + std::string offset; + uint64_t mode = 0; + if (expression.kind == "bitfield_extract") + mode = expression.predicate == "signed" ? 5 : 0; + else if (expression.kind == "bitfield_popcount") + mode = 6; + else if (expression.kind == "bitfield_clz") + mode = 7; + else if (expression.kind == "bitfield_ctz") + mode = 8; + else if (expression.kind == "bitfield_clear") + mode = 1; + else if (expression.kind == "bitfield_set") + mode = 2; + else if (expression.kind == "bitfield_reverse_bytes") + mode = 4; + else { + if (expression.operands.size() != 4) + return pycError("ALU bitfield insert arity mismatch"); + auto inserted = value(expression.operands[1]); + width = value(expression.operands[2]); + auto insertOffset = value(expression.operands[3]); + if (!inserted || !width || !insertOffset) + return pycError("ALU bitfield insert references unknown value"); + source = *inserted; + offset = emitOffset(*insertOffset); + mode = 3; + } + if (expression.kind != "bitfield_insert") + offset = emitOffset(*offsetRaw); + result = emitWrappingBitfield(source, *width, offset, mode); + } } else if (expression.kind == "bit_extract" || expression.kind == "aggregate_get") { if (expression.operands.size() != 1 || expression.width == 0) @@ -2225,7 +2458,8 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, << "\n"; } else if (expression.kind == "add" || expression.kind == "sub" || expression.kind == "mul" || expression.kind == "udiv" || - expression.kind == "urem" || expression.kind == "and" || + expression.kind == "sdiv" || expression.kind == "urem" || + expression.kind == "srem" || expression.kind == "and" || expression.kind == "or" || expression.kind == "xor") { result = newValue(); if (expression.operands.size() != 2) @@ -2379,10 +2613,10 @@ emitTransform(const QueueGraphPlan &plan, const QueueBlockPlan &block, if (sourceLocations) { const std::string suffix = sourceLocationSuffix(effectiveProvenance); if (!suffix.empty()) - for (unsigned valueIndex = firstGeneratedValue; - valueIndex < nextValue; ++valueIndex) - sourceLocations->try_emplace( - "%v" + std::to_string(valueIndex), suffix); + for (unsigned valueIndex = firstGeneratedValue; valueIndex < nextValue; + ++valueIndex) + sourceLocations->try_emplace("%v" + std::to_string(valueIndex), + suffix); } } if (yieldIndex >= block.yields.size()) { @@ -2750,6 +2984,8 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { llvm::StringMap routeCondition; llvm::StringMap selectStates; llvm::StringMap atomicTransformValid; + llvm::StringMap>> + transformExpressionValues; llvm::StringMap firingPresence; llvm::StringMap firingGuard; llvm::StringMap firingAccepted; @@ -3044,14 +3280,24 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { body << " " << producerValid << " = pyc.wire : i1\n"; atomicTransformValid[queue.name] = producerValid; } + auto cached = transformExpressionValues.find(queue.name); + if (cached == transformExpressionValues.end()) { + auto values = std::make_shared>(); + auto emitted = emitTransform( + plan, transform, inputDataValues, inputTypes, producer.index, + nextValue, body, values.get(), nullptr, nullptr, {}, nullptr, + nullptr, nullptr, &sourceLocations); + if (!emitted) + return emitted.takeError(); + for (const std::string &output : transform.outputs) + transformExpressionValues[output] = values; + cached = transformExpressionValues.find(queue.name); + } auto transformed = - emitTransform(plan, transform, inputDataValues, inputTypes, - producer.index, nextValue, body, nullptr, nullptr, - nullptr, {}, nullptr, nullptr, nullptr, - &sourceLocations); - if (!transformed) - return transformed.takeError(); - producerData = std::move(*transformed); + cached->getValue()->find(transform.yields[producer.index]); + if (transformed == cached->getValue()->end()) + return pycError("transform expression identity is missing"); + producerData = transformed->getValue(); } else if (firingProducer != firingByOutput.end()) { const TransformProducer &producer = firingProducer->getValue(); const QueueBlockPlan &firing = *producer.block; @@ -3073,12 +3319,11 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { auto cached = firingExpressionValues.find(queue.name); if (cached == firingExpressionValues.end()) { auto values = std::make_shared>(); - auto emitted = - emitTransform(plan, firing, inputDataValues, inputTypes, - producer.index, nextValue, body, values.get(), - &tableStateValues, &roundRobinStates, firing.name, - nullptr, nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto emitted = emitTransform( + plan, firing, inputDataValues, inputTypes, producer.index, + nextValue, body, values.get(), &tableStateValues, + &roundRobinStates, firing.name, nullptr, nullptr, + &sharedTableExpressionValues, &sourceLocations); if (!emitted) return emitted.takeError(); for (const std::string &output : firing.outputs) @@ -3132,11 +3377,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { inputTypes.push_back(inputQueue->payloadType); } auto values = std::make_shared>(); - auto index = - emitTransform(plan, read, inputDataValues, inputTypes, 0, nextValue, - body, values.get(), &tableStateValues, nullptr, {}, - nullptr, nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto index = emitTransform( + plan, read, inputDataValues, inputTypes, 0, nextValue, body, + values.get(), &tableStateValues, nullptr, {}, nullptr, nullptr, + &sharedTableExpressionValues, &sourceLocations); if (!index) return index.takeError(); auto present = values->find(read.yields[1]); @@ -3246,11 +3490,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { evaluation.yields.insert(evaluation.yields.end(), validNames.begin(), validNames.end()); llvm::StringMap emittedValues; - auto emitted = emitTransform(plan, evaluation, {}, {}, 0, nextValue, - body, &emittedValues, &tableStateValues, - &roundRobinStates, group.name, nullptr, - nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto emitted = emitTransform( + plan, evaluation, {}, {}, 0, nextValue, body, &emittedValues, + &tableStateValues, &roundRobinStates, group.name, nullptr, + nullptr, &sharedTableExpressionValues, &sourceLocations); if (!emitted) return emitted.takeError(); TableReadGroupState created; @@ -3341,11 +3584,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { return pycError("route input is not available in topological order"); auto selector = routeSelector.find(route.name); if (selector == routeSelector.end()) { - auto selected = - emitTransform(plan, route, {data->getValue()}, - {inputQueue->payloadType}, 0, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto selected = emitTransform( + plan, route, {data->getValue()}, {inputQueue->payloadType}, 0, + nextValue, body, nullptr, nullptr, nullptr, {}, nullptr, nullptr, + nullptr, &sourceLocations); if (!selected) return selected.takeError(); routeSelector[route.name] = *selected; @@ -3373,11 +3615,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { controlData == outputData.end() || !controlQueue) return pycError( "select control is not available in topological order"); - auto selector = - emitTransform(plan, select, {controlData->getValue()}, - {controlQueue->payloadType}, 0, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto selector = emitTransform( + plan, select, {controlData->getValue()}, + {controlQueue->payloadType}, 0, nextValue, body, nullptr, nullptr, + nullptr, {}, nullptr, nullptr, nullptr, &sourceLocations); if (!selector) return selector.takeError(); auto selectorType = yieldedType(select, select.yields.front(), @@ -3522,11 +3763,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { return dataType.takeError(); if (!dataWidth) return dataWidth.takeError(); - auto costValue = - emitTransform(plan, credit, {inputDataValue->getValue()}, - {inputQueue->payloadType}, 0, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto costValue = emitTransform( + plan, credit, {inputDataValue->getValue()}, + {inputQueue->payloadType}, 0, nextValue, body, nullptr, nullptr, + nullptr, {}, nullptr, nullptr, nullptr, &sourceLocations); if (!costValue) return costValue.takeError(); auto costType = @@ -3796,11 +4036,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { auto dataType = pycType(plan, inputQueue->payloadType); if (!dataType) return dataType.takeError(); - auto keyValue = - emitTransform(plan, reorder, {inputDataValue->getValue()}, - {inputQueue->payloadType}, 0, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto keyValue = emitTransform( + plan, reorder, {inputDataValue->getValue()}, + {inputQueue->payloadType}, 0, nextValue, body, nullptr, nullptr, + nullptr, {}, nullptr, nullptr, nullptr, &sourceLocations); if (!keyValue) return keyValue.takeError(); auto keyType = yieldedType(reorder, reorder.yields.front(), @@ -3957,19 +4196,17 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { emitBinary("or", state.valid, inputValidValue->getValue(), "i1"); state.selectedIteration = emitMux(state.valid, state.iteration, zeroIteration, iterationType); - auto updated = - emitTransform(plan, feedback, {selectedData}, - {inputQueue->payloadType}, 0, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto updated = emitTransform( + plan, feedback, {selectedData}, {inputQueue->payloadType}, 0, + nextValue, body, nullptr, nullptr, nullptr, {}, nullptr, nullptr, + nullptr, &sourceLocations); if (!updated) return updated.takeError(); state.updated = std::move(*updated); - auto condition = - emitTransform(plan, feedback, {selectedData}, - {inputQueue->payloadType}, 1, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto condition = emitTransform( + plan, feedback, {selectedData}, {inputQueue->payloadType}, 1, + nextValue, body, nullptr, nullptr, nullptr, {}, nullptr, nullptr, + nullptr, &sourceLocations); if (!condition) return condition.takeError(); auto conditionType = @@ -4073,11 +4310,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { endpointValids.push_back(valid->getValue()); endpointData.push_back(data->getValue()); for (size_t policy = 0; policy < 3; ++policy) { - auto value = - emitTransform(plan, *endpoint, {data->getValue()}, - {input->payloadType}, policy, nextValue, body, - nullptr, nullptr, nullptr, {}, nullptr, nullptr, - nullptr, &sourceLocations); + auto value = emitTransform(plan, *endpoint, {data->getValue()}, + {input->payloadType}, policy, nextValue, + body, nullptr, nullptr, nullptr, {}, nullptr, + nullptr, nullptr, &sourceLocations); if (!value) return value.takeError(); auto yielded = yieldedType(*endpoint, endpoint->yields[policy], @@ -4311,11 +4547,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { inputTypes.push_back(inputQueue->payloadType); } auto values = std::make_shared>(); - auto index = - emitTransform(plan, block, inputDataValues, inputTypes, 0, nextValue, - body, values.get(), &tableStateValues, nullptr, {}, - nullptr, nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto index = emitTransform( + plan, block, inputDataValues, inputTypes, 0, nextValue, body, + values.get(), &tableStateValues, nullptr, {}, nullptr, nullptr, + &sharedTableExpressionValues, &sourceLocations); if (!index) return index.takeError(); auto present = values->find(block.yields[1]); @@ -4328,10 +4563,10 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { } else if (block.kind == "table_masked_write") { QueueBlockPlan maskBlock = expressionSlice(block, block.yields[0]); QueueBlockPlan enableBlock = expressionSlice(block, block.yields[1]); - auto mask = emitTransform(plan, maskBlock, {}, {}, 0, nextValue, body, - nullptr, &tableStateValues, nullptr, {}, - nullptr, nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto mask = + emitTransform(plan, maskBlock, {}, {}, 0, nextValue, body, nullptr, + &tableStateValues, nullptr, {}, nullptr, nullptr, + &sharedTableExpressionValues, &sourceLocations); auto enabled = emitTransform(plan, enableBlock, {}, {}, 0, nextValue, body, nullptr, &tableStateValues, nullptr, {}, nullptr, nullptr, @@ -5133,9 +5368,9 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { return pycError("Table register-bank state is missing"); // A Table may carry field-mode and replace-mode firing writes together, but // only when every conflicting endpoint declares explicit writer arbitration - // (see VerifyValueConstraints). Both passes are emitted in gfsim's order -- - // every FieldMerge against the committed image first, then every Replace on - // top of that -- so the commit order never depends on plan block order. + // (see VerifyValueConstraints). Both passes are emitted in gfsim's order + // -- every FieldMerge against the committed image first, then every Replace + // on top of that -- so the commit order never depends on plan block order. bool hasFiringFieldWrite = false; bool hasFiringReplaceWrite = false; for (const QueueBlockPlan &block : plan.blocks) { @@ -5286,11 +5521,11 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { arbitration != blockArbitrationAllowed.end()) selected = emitBinary("and", selected, arbitration->getValue(), "i1"); QueueBlockPlan valueBlock = expressionSlice(block, block.yields[2]); - auto proposed = emitTransform( - plan, valueBlock, {state->getValue().value[slot]}, - {table.entryType}, 0, nextValue, body, nullptr, &tableStateValues, - nullptr, {}, nullptr, nullptr, &sharedTableExpressionValues, - &sourceLocations); + auto proposed = + emitTransform(plan, valueBlock, {state->getValue().value[slot]}, + {table.entryType}, 0, nextValue, body, nullptr, + &tableStateValues, nullptr, {}, nullptr, nullptr, + &sharedTableExpressionValues, &sourceLocations); if (!proposed) return proposed.takeError(); auto merged = emitTableFieldMerge(next, *proposed, table.entryType, @@ -5441,8 +5676,8 @@ llvm::Expected generateQueueGraphPyc(const QueueGraphPlan &plan) { return sourceMap.takeError(); output << "module attributes {pyc.top = @" << top.str() << ", pyc.frontend.contract = \"pycircuit\", pyc.source_map = " - << mlirStringLiteral(*sourceMap) << "} {\n func.func @" - << top.str() << '('; + << mlirStringLiteral(*sourceMap) << "} {\n func.func @" << top.str() + << '('; writeList(output, arguments); output << ") -> ("; writeList(output, resultTypes); diff --git a/compiler/acir/lib/Dialect/ACIR/ACIROps.cpp b/compiler/acir/lib/Dialect/ACIR/ACIROps.cpp index 99be1ca90..c592f13e7 100644 --- a/compiler/acir/lib/Dialect/ACIR/ACIROps.cpp +++ b/compiler/acir/lib/Dialect/ACIR/ACIROps.cpp @@ -302,8 +302,8 @@ verifyTypedRuleSummary(Operation *operation, ValueRange inputs, effect.set("kind", RuleEffectKindAttr::get(operation->getContext(), RuleEffectKind::StateRead)); effect.set("resource", get.getSlotAttr()); - effect.set("guard_kind", RuleGuardKindAttr::get( - operation->getContext(), RuleGuardKind::Always)); + effect.set("guard_kind", RuleGuardKindAttr::get(operation->getContext(), + RuleGuardKind::Always)); expectedEffects.push_back(builder.getDictionaryAttr(effect)); }); body.walk([&](SlotProposeReleaseOp release) { @@ -311,9 +311,9 @@ verifyTypedRuleSummary(Operation *operation, ValueRange inputs, effect.set("kind", RuleEffectKindAttr::get(operation->getContext(), RuleEffectKind::StateWrite)); effect.set("resource", release.getSlotAttr()); - effect.set("guard_kind", RuleGuardKindAttr::get( - operation->getContext(), - guardKindFor(release.getWhen()))); + effect.set("guard_kind", + RuleGuardKindAttr::get(operation->getContext(), + guardKindFor(release.getWhen()))); expectedEffects.push_back(builder.getDictionaryAttr(effect)); }); @@ -635,11 +635,12 @@ LogicalResult RuleOp::verify() { return emitOpError("permits at most one functional condition"); SmallVector outputPaths; getBody().walk([&](RuleOutputOp output) { outputPaths.push_back(output); }); - const bool hasPathEvidence = getOutputs().size() > 1 || - !outputPaths.empty() || - llvm::any_of(proposals, [](TableProposeOp op) { - return static_cast(op.getWhen()); - }) || !slotReleases.empty(); + const bool hasPathEvidence = + getOutputs().size() > 1 || !outputPaths.empty() || + llvm::any_of( + proposals, + [](TableProposeOp op) { return static_cast(op.getWhen()); }) || + !slotReleases.empty(); if (hasPathEvidence) { if (conditions != 1) return emitOpError("SSA path evidence requires one rule condition"); @@ -884,11 +885,12 @@ LogicalResult SourceOp::verify() { if (!declaration || !active.insert(declaration).second) return false; auto fields = declaration->getAttrOfType("fields"); - const bool found = fields && llvm::any_of(fields, [&](Attribute rawField) { - auto field = dyn_cast(rawField); - auto fieldType = field ? field.getAs("type") : TypeAttr(); - return fieldType && containsDeclaredRange(fieldType.getValue()); - }); + const bool found = + fields && llvm::any_of(fields, [&](Attribute rawField) { + auto field = dyn_cast(rawField); + auto fieldType = field ? field.getAs("type") : TypeAttr(); + return fieldType && containsDeclaredRange(fieldType.getValue()); + }); active.erase(declaration); return found; }; @@ -1292,9 +1294,8 @@ LogicalResult FiringOp::verify() { SmallVector conditions; getBody().walk( [&](TableProposeOp proposal) { proposals.push_back(proposal); }); - getBody().walk([&](SlotProposeReleaseOp release) { - slotReleases.push_back(release); - }); + getBody().walk( + [&](SlotProposeReleaseOp release) { slotReleases.push_back(release); }); getBody().walk([&](TableGetOp read) { tableReads.push_back(read); }); getBody().walk( [&](FiringConditionOp condition) { conditions.push_back(condition); }); @@ -1317,11 +1318,12 @@ LogicalResult FiringOp::verify() { if (output.getOrdinal() < 0 || static_cast(output.getOrdinal()) >= getOutputs().size()) return output.emitOpError("ordinal must name one firing output"); - const bool hasPathEvidence = getOutputs().size() > 1 || - !outputPaths.empty() || - llvm::any_of(proposals, [](TableProposeOp op) { - return static_cast(op.getWhen()); - }) || !slotReleases.empty(); + const bool hasPathEvidence = + getOutputs().size() > 1 || !outputPaths.empty() || + llvm::any_of( + proposals, + [](TableProposeOp op) { return static_cast(op.getWhen()); }) || + !slotReleases.empty(); if (hasPathEvidence) { if (conditions.size() != 1) return emitOpError("SSA path evidence requires one firing condition"); @@ -1432,9 +1434,8 @@ LogicalResult FiringOp::verify() { "inferred footprint must exactly match its state operation"); } } - const bool validArity = - !getInputs().empty() || !getOutputs().empty() || !proposals.empty() || - !slotReleases.empty(); + const bool validArity = !getInputs().empty() || !getOutputs().empty() || + !proposals.empty() || !slotReleases.empty(); if (conditions.empty() && requiresInferredSchedule) { return emitOpError("requires one typed functional condition"); } @@ -2010,10 +2011,9 @@ LogicalResult VarConstantOp::verify() { if (auto range = dyn_cast(result.getElementType())) { auto integer = dyn_cast_or_null(value); const uint64_t upper = range.getUpper(); - const unsigned width = - upper == std::numeric_limits::max() - ? 64 - : std::max(1u, llvm::Log2_64_Ceil(upper + 1)); + const unsigned width = upper == std::numeric_limits::max() + ? 64 + : std::max(1u, llvm::Log2_64_Ceil(upper + 1)); if (!integer || !integer.getType().isSignlessInteger(width) || integer.getValue().getZExtValue() < range.getLower() || integer.getValue().getZExtValue() > upper) @@ -2181,7 +2181,8 @@ LogicalResult VarWithElementOp::verify() { return emitOpError("index must be an unsigned scalar"); if (range && range.getUpper() >= static_cast(array.getLength())) return emitOpError("bounded index exceeds the value_array length"); - if (getValue().getType() != VarType::get(getContext(), array.getElementType())) + if (getValue().getType() != + VarType::get(getContext(), array.getElementType())) return emitOpError("replacement must match the value_array element type"); if (getResult().getType() != getAggregate().getType()) return emitOpError("result must preserve the value_array type"); @@ -2203,8 +2204,8 @@ static bool supportsZeroImage(Operation *operation, Type type, Operation *declaration = recordDecl(operation, type); if (!declaration || !seen.insert(declaration).second) return false; - const bool supported = llvm::all_of( - declarationFields(declaration), [&](Attribute rawField) { + const bool supported = + llvm::all_of(declarationFields(declaration), [&](Attribute rawField) { return supportsZeroImage( operation, fieldType(cast(rawField)), seen); }); @@ -2241,10 +2242,9 @@ LogicalResult VarDeclOp::verify() { const bool zeroImage = zero && zero.getValue().isZero(); if (auto range = dyn_cast(getValueType())) { const uint64_t upper = range.getUpper(); - const unsigned width = - upper == std::numeric_limits::max() - ? 64 - : std::max(1u, llvm::Log2_64_Ceil(upper + 1)); + const unsigned width = upper == std::numeric_limits::max() + ? 64 + : std::max(1u, llvm::Log2_64_Ceil(upper + 1)); if (!zero || !zero.getType().isSignlessInteger(width) || zero.getValue().getZExtValue() < range.getLower() || zero.getValue().getZExtValue() > upper) @@ -2255,8 +2255,8 @@ LogicalResult VarDeclOp::verify() { if ((!init || init.getType() != getValueType()) && !(zeroImage && isa(getValueType()) && supportsZeroImage(*this, getValueType(), seen))) - return emitOpError( - "init must match value type or be the zero image for a struct or enum"); + return emitOpError("init must match value type or be the zero image for " + "a struct or enum"); } if (getOwner().empty() || !getOwner().starts_with('/') || (getOwner().size() > 1 && getOwner().ends_with('/'))) @@ -2556,10 +2556,44 @@ LogicalResult VarUDivOp::verify() { return verifyVarUnsignedBinary(*this, getLhs(), getRhs(), getResult()); } +LogicalResult VarSDivOp::verify() { + return verifyVarUnsignedBinary(*this, getLhs(), getRhs(), getResult()); +} + LogicalResult VarURemOp::verify() { return verifyVarUnsignedBinary(*this, getLhs(), getRhs(), getResult()); } +LogicalResult VarSRemOp::verify() { + return verifyVarUnsignedBinary(*this, getLhs(), getRhs(), getResult()); +} + +LogicalResult VarDivRemOp::verify() { + auto lhs = dyn_cast(getLhs().getType()); + auto rhs = dyn_cast(getRhs().getType()); + auto signedMode = dyn_cast(getSignedMode().getType()); + auto wordMode = dyn_cast(getWordMode().getType()); + auto quotient = dyn_cast(getQuotient().getType()); + auto remainder = dyn_cast(getRemainder().getType()); + if (!lhs || !rhs || !signedMode || !wordMode || !quotient || !remainder || + lhs != rhs) + return emitOpError("divrem operands and results must be ac.var values"); + auto lhsInt = dyn_cast(lhs.getElementType()); + auto rhsInt = dyn_cast(rhs.getElementType()); + auto quotientInt = dyn_cast(quotient.getElementType()); + auto remainderInt = dyn_cast(remainder.getElementType()); + auto signedModeInt = dyn_cast(signedMode.getElementType()); + auto wordModeInt = dyn_cast(wordMode.getElementType()); + if (!lhsInt || !rhsInt || !quotientInt || !remainderInt || !signedModeInt || + !wordModeInt || !lhsInt.isSignless() || !rhsInt.isSignless() || + !signedModeInt.isSignless() || !wordModeInt.isSignless() || + signedModeInt.getWidth() != 1 || wordModeInt.getWidth() != 1 || + lhsInt.getWidth() != 64 || rhsInt.getWidth() != 64 || + quotientInt.getWidth() != 64 || remainderInt.getWidth() != 64) + return emitOpError("divrem currently requires i64 operands and results"); + return success(); +} + namespace { template @@ -2569,7 +2603,8 @@ struct CanonicalizeUnsignedPowerOfTwo final : OpRewritePattern { LogicalResult matchAndRewrite(SourceOp operation, PatternRewriter &rewriter) const override { auto divisor = operation.getRhs().template getDefiningOp(); - auto value = divisor ? dyn_cast(divisor.getValue()) : IntegerAttr(); + auto value = + divisor ? dyn_cast(divisor.getValue()) : IntegerAttr(); if (!value || !value.getValue().isPowerOf2()) return failure(); auto resultType = cast(operation.getResult().getType()); @@ -2578,11 +2613,11 @@ struct CanonicalizeUnsignedPowerOfTwo final : OpRewritePattern { const uint64_t replacementValue = IsRemainder ? divisorValue - 1 : llvm::Log2_64(divisorValue); auto replacementConstant = VarConstantOp::create( - rewriter, - operation.getLoc(), resultType, + rewriter, operation.getLoc(), resultType, rewriter.getIntegerAttr(integerType, replacementValue)); - auto replacement = TargetOp::create(rewriter, operation.getLoc(), resultType, - operation.getLhs(), replacementConstant); + auto replacement = + TargetOp::create(rewriter, operation.getLoc(), resultType, + operation.getLhs(), replacementConstant); replacement->setDiscardableAttrs(operation->getDiscardableAttrDictionary()); rewriter.replaceOp(operation, replacement.getResult()); return success(); @@ -3021,8 +3056,8 @@ static bool isUnsignedScalar(Type type) { static LogicalResult verifyRangeConversion(Operation *operation, Value input, Value result) { Type inputElement = cast(input.getType()).getElementType(); - auto resultRange = dyn_cast( - cast(result.getType()).getElementType()); + auto resultRange = + dyn_cast(cast(result.getType()).getElementType()); if (!isUnsignedScalar(inputElement) || !resultRange) return operation->emitOpError( "range conversion requires unsigned scalar input and range result"); @@ -3051,8 +3086,8 @@ LogicalResult VarRangeRefineOp::verify() { } LogicalResult VarRangeBitsOp::verify() { - auto inputRange = dyn_cast( - cast(getInput().getType()).getElementType()); + auto inputRange = + dyn_cast(cast(getInput().getType()).getElementType()); auto resultInteger = dyn_cast( cast(getResult().getType()).getElementType()); if (!inputRange || !resultInteger || !resultInteger.isSignless() || @@ -3065,12 +3100,12 @@ LogicalResult VarRangeBitsOp::verify() { static LogicalResult verifyRangeArithmetic(Operation *operation, Value lhs, Value rhs, Value result, bool subtract) { - auto left = dyn_cast( - cast(lhs.getType()).getElementType()); - auto right = dyn_cast( - cast(rhs.getType()).getElementType()); - auto actual = dyn_cast( - cast(result.getType()).getElementType()); + auto left = + dyn_cast(cast(lhs.getType()).getElementType()); + auto right = + dyn_cast(cast(rhs.getType()).getElementType()); + auto actual = + dyn_cast(cast(result.getType()).getElementType()); if (!left || !right || !actual) return operation->emitOpError( "bounded arithmetic requires range operands and result"); @@ -3136,6 +3171,144 @@ LogicalResult VarInsertOp::verify() { getLsb(), value.getWidth()); } +static LogicalResult verifyAluInteger(Operation *operation, Value value, + unsigned expectedWidth = 0) { + auto type = + dyn_cast(cast(value.getType()).getElementType()); + if (!type || !type.isSignless() || type.getWidth() == 0 || + type.getWidth() > 64) + return operation->emitOpError("operand must be a signless i1..i64 Var"); + if (expectedWidth != 0 && type.getWidth() != expectedWidth) + return operation->emitOpError() + << "operand width must be " << expectedWidth; + return success(); +} + +static LogicalResult verifyAluBinary(Operation *operation, Value lhs, Value rhs, + Value result, unsigned expectedWidth = 0) { + if (lhs.getType() != rhs.getType() || lhs.getType() != result.getType()) + return operation->emitOpError( + "operands and result must have one identical Var type"); + return verifyAluInteger(operation, lhs, expectedWidth); +} + +static LogicalResult verifyAluTernary(Operation *operation, Value lhs, + Value rhs, Value aux, Value result, + unsigned expectedWidth = 0) { + if (lhs.getType() != rhs.getType() || lhs.getType() != aux.getType() || + lhs.getType() != result.getType()) + return operation->emitOpError( + "operands and result must have one identical Var type"); + return verifyAluInteger(operation, lhs, expectedWidth); +} + +static LogicalResult verifyBitfieldControls(Operation *operation, Value value, + Value width, Value offset, + Value result) { + if (value.getType() != result.getType()) + return operation->emitOpError( + "value and result must have one identical Var type"); + if (failed(verifyAluInteger(operation, value, 64)) || + failed(verifyAluInteger(operation, width, 7)) || + failed(verifyAluInteger(operation, offset, 6))) + return failure(); + return success(); +} + +#define DEFINE_ALU_BINARY_VERIFY(CLASS, WIDTH) \ + LogicalResult CLASS::verify() { \ + return verifyAluBinary(*this, getLhs(), getRhs(), getResult(), WIDTH); \ + } + +DEFINE_ALU_BINARY_VERIFY(VarAddwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSubwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarAndwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarOrwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarXorwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSllOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSrlOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSraOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSllwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSrlwOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSrawOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSminOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarUminOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarSmaxOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarUmaxOp, 64) +DEFINE_ALU_BINARY_VERIFY(VarMulwOp, 64) + +#undef DEFINE_ALU_BINARY_VERIFY + +LogicalResult VarMaddOp::verify() { + return verifyAluTernary(*this, getLhs(), getRhs(), getAux(), getResult(), 64); +} + +LogicalResult VarMaddwOp::verify() { + return verifyAluTernary(*this, getLhs(), getRhs(), getAux(), getResult(), 64); +} + +LogicalResult VarMsubOp::verify() { + return verifyAluTernary(*this, getLhs(), getRhs(), getAux(), getResult(), 64); +} + +LogicalResult VarBitfieldExtractOp::verify() { + return verifyBitfieldControls(*this, getValue(), getWidth(), getOffset(), + getResult()); +} + +#define DEFINE_BITFIELD_VERIFY(CLASS) \ + LogicalResult CLASS::verify() { \ + return verifyBitfieldControls(*this, getValue(), getWidth(), getOffset(), \ + getResult()); \ + } + +DEFINE_BITFIELD_VERIFY(VarBitfieldPopcountOp) +DEFINE_BITFIELD_VERIFY(VarBitfieldClzOp) +DEFINE_BITFIELD_VERIFY(VarBitfieldCtzOp) +DEFINE_BITFIELD_VERIFY(VarBitfieldClearOp) +DEFINE_BITFIELD_VERIFY(VarBitfieldSetOp) +DEFINE_BITFIELD_VERIFY(VarBitfieldReverseBytesOp) + +#undef DEFINE_BITFIELD_VERIFY + +LogicalResult VarBitfieldInsertOp::verify() { + if (getValue().getType() != getSource().getType() || + getValue().getType() != getResult().getType()) + return emitOpError( + "value, source, and result must have one identical Var type"); + return verifyBitfieldControls(*this, getValue(), getWidth(), getOffset(), + getResult()); +} + +LogicalResult VarSextLowOp::verify() { + if (getValue().getType() != getResult().getType()) + return emitOpError("value and result must have one identical Var type"); + if (failed(verifyAluInteger(*this, getValue(), 64))) + return failure(); + return verifyAluInteger(*this, getWidth(), 7); +} + +LogicalResult VarZextLowOp::verify() { + if (getValue().getType() != getResult().getType()) + return emitOpError("value and result must have one identical Var type"); + if (failed(verifyAluInteger(*this, getValue(), 64))) + return failure(); + return verifyAluInteger(*this, getWidth(), 7); +} + +LogicalResult VarCselOp::verify() { + if (getLhs().getType() != getRhs().getType() || + getLhs().getType() != getResult().getType()) + return emitOpError( + "selected values and result must have one identical Var type"); + if (failed(verifyAluInteger(*this, getLhs(), 64))) + return failure(); + if (failed(verifyAluInteger(*this, getPredicate(), 1)) || + failed(verifyAluInteger(*this, getNegateFalse(), 1))) + return emitOpError("predicate and negate_false must be !ac.var"); + return success(); +} + namespace { Location projectionSourceLocation(Location location) { @@ -4023,9 +4196,8 @@ LogicalResult TableIndexOp::verify() { Type element = cast(coordinate.getType()).getElementType(); auto type = dyn_cast(element); auto range = dyn_cast(element); - if ((!range && - (!type || !type.isSignless() || - type.getWidth() != canonicalTableIndexWidth(extent))) || + if ((!range && (!type || !type.isSignless() || + type.getWidth() != canonicalTableIndexWidth(extent))) || (range && range.getUpper() >= static_cast(extent))) return emitOpError( "coordinate type must use the canonical unsigned axis width or a " diff --git a/compiler/acir/tools/acir-queue-veriloggen.py b/compiler/acir/tools/acir-queue-veriloggen.py index 60b019d0f..991cae705 100644 --- a/compiler/acir/tools/acir-queue-veriloggen.py +++ b/compiler/acir/tools/acir-queue-veriloggen.py @@ -20,9 +20,9 @@ import subprocess import sys import tempfile -from collections.abc import Iterable from dataclasses import dataclass from pathlib import Path +from typing import Iterable class PYCVerilogError(ValueError): @@ -182,11 +182,15 @@ def _port_decl(direction: str, value: Value) -> str: def _literal(value: str, width: int) -> str: + if value == "true": + return f"{width}'d1" + if value == "false": + return f"{width}'d0" try: number = int(value, 0) except ValueError as exc: raise PYCVerilogError(f"unsupported PYC constant {value}") from exc - return f"{width}'d{number}" + return f"{width}'d{number % (1 << width)}" def _parse_attr_int(attrs: str, key: str) -> int: @@ -196,12 +200,81 @@ def _parse_attr_int(attrs: str, key: str) -> int: return int(match.group(1)) -def _runtime_sources(runtime_dir: Path) -> Iterable[str]: - for name in ("pyc_reg.v", "pyc_fifo.v"): +def _runtime_sources( + runtime_dir: Path, + *, + include_fifo: bool = False, + include_reg: bool = False, + include_popcount: bool = False, + include_rr_arbiter: bool = False, + include_wrapping_bitfield: bool = False, + include_divrem: bool = False, + include_priority: bool = False, +) -> Iterable[str]: + names: list[str] = [] + if include_reg: + names.append("pyc_reg.v") + if include_fifo: + names.append("pyc_fifo.v") + if include_popcount: + names.append("pyc_popcount.v") + if include_rr_arbiter: + names.append("pyc_rr_arbiter.v") + if include_wrapping_bitfield: + names.append("pyc_runtime_wrapping_bitfield.sv") + if include_divrem: + # Inline the qualified PTO wrapper and its small BaseJump dependency + # closure. The generated design therefore consumes one physical + # iterative divider instead of synthesizing a hidden / or % operator. + names.extend( + [ + "div_pto_v2/div_pto_v2/basejump/bsg_defines.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_dff_en.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_mux_one_hot.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_xnor.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_nor2.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_xor.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_adder_cin.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_counter_clear_up.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative_controller.sv", + "div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative.sv", + "div_pto_v2/div_pto_v2/pyc_word_operand_normalize.sv", + "div_pto_v2/div_pto_v2/pyc_word_result_normalize.sv", + "div_pto_v2/div_pto_v2/pyc_div_special_cases.sv", + "div_pto_v2/div_pto_v2/pyc_runtime_div.sv", + "div_pto_v2/div_pto_v2/pyc_runtime_div_packet.sv", + ] + ) + if include_priority: + # Inline the small BaseJump dependency closure so the generated file + # remains directly consumable without an extra -I search path. + names.extend( + [ + "basejump/bsg_defines.sv", + "basejump/bsg_scan.sv", + "basejump/bsg_encode_one_hot.sv", + "basejump/bsg_priority_encode_one_hot_out.sv", + "basejump/bsg_priority_encode.sv", + "pyc_runtime_basejump_priority_encode.v", + ] + ) + for name in names: path = runtime_dir / name if not path.is_file(): raise PYCVerilogError(f"missing in-tree PYC runtime module: {path}") - yield f"// --- PYC runtime: {path.name}\n{path.read_text(encoding='utf-8')}" + content = path.read_text(encoding="utf-8") + if ( + (include_priority or include_divrem) + and path.suffix in {".sv", ".v"} + and "basejump/" in name + ): + # Dependencies are concatenated in topological order above; their + # relative include directives would otherwise refer to files that + # are no longer adjacent to the generated artifact. + content = re.sub( + r'^\s*`include\s+"[^"]+"\s*$', "", content, flags=re.MULTILINE + ) + yield f"// --- PYC runtime: {name}\n{content}" def emit_verilog(module: Module, runtime_dir: Path) -> str: @@ -212,6 +285,12 @@ def emit_verilog(module: Module, runtime_dir: Path) -> str: assertions: list[str] = [] returns: list[str] | None = None seen_outputs: set[str] = set() + # Queue FIFOs provide the transaction boundary around pure PYC values. + # A variable-latency divider must retain the packet while its iterative + # result is in flight, so remember the FIFO control nets and patch the + # corresponding ready/valid connections when the divrem op is parsed. + fifo_payloads: list[dict[str, str]] = [] + divrem_adapter: dict[str, str] | None = None def add_value(name: str, type_: str) -> Value: value = Value(name, type_) @@ -307,13 +386,34 @@ def add_value(name: str, type_: str) -> Value: _value(values, inputs[4]), ) clk, rst = _value(values, inputs[0]), _value(values, inputs[1]) + fifo_response_path = ( + divrem_adapter and divrem_adapter["input_valid_name"] == in_valid.name + ) + fifo_in_valid_net = ( + divrem_adapter["response_valid"] if fifo_response_path else in_valid.net + ) instances.append( f" pyc_fifo #(.WIDTH({out_values[2].width}), .DEPTH({depth})) fifo_{out_values[0].net} (\n" f" .clk({clk.net}), .rst({rst.net}),\n" - f" .in_valid({in_valid.net}), .in_ready({out_values[0].net}), .in_data({in_data.net}),\n" - f" .out_valid({out_values[1].net}), .out_ready({out_ready.net}), .out_data({out_values[2].net})\n" + f" .in_valid({fifo_in_valid_net}), .in_ready({out_values[0].net}), .in_data({in_data.net}),\n" + f" .out_valid({out_values[1].net}), .out_ready({(divrem_adapter['request_ready'] if divrem_adapter and divrem_adapter['input_ready_name'] == out_ready.name else out_ready.net)}), .out_data({out_values[2].net})\n" f" );" ) + fifo_payloads.append( + { + "payload_name": out_values[2].name, + "out_valid_name": out_values[1].name, + "out_ready_name": out_ready.name, + } + ) + # The output FIFO is parsed after divrem. Its input-valid is the + # response side of the adapter, and its in-ready feeds the + # adapter's response-ready wire. + if fifo_response_path: + divrem_adapter["response_ready_source"] = out_values[0].net + assigns.append( + f" assign {divrem_adapter['response_ready']} = {out_values[0].net};" + ) continue if rhs.startswith("pyc.reg "): @@ -362,6 +462,151 @@ def add_value(name: str, type_: str) -> Value: ) continue + if rhs.startswith("pyc.priority_encode "): + match = re.fullmatch( + r"pyc\.priority_encode\s+(.*?)\s*\{(.*?)\}\s*:\s*(\S+)\s*->\s*(\S+)", + rhs, + ) + if not match or len(lhs) != 1: + raise PYCVerilogError(f"cannot parse priority encoder: {line}") + inputs = _ssa_names(match.group(1)) + if len(inputs) != 1: + raise PYCVerilogError(f"priority encoder expects one operand: {line}") + src = _value(values, inputs[0]) + out = add_value(lhs[0], match.group(4)) + width = _parse_attr_int(match.group(2), "width") + lo_to_hi = _parse_attr_int(match.group(2), "lo_to_hi") + if width != src.width or width <= 0: + raise PYCVerilogError("priority encoder width must match input type") + index_width = max(1, (width - 1).bit_length()) + if out.width != index_width + 1: + raise PYCVerilogError("priority encoder result must be {valid,index}") + index_net = f"priority_index_{out.net}" + valid_net = f"priority_valid_{out.net}" + declarations.append(f" wire [{index_width - 1}:0] {index_net};") + declarations.append(f" wire {valid_net};") + instances.append( + f" pyc_runtime_basejump_priority_encode #(.WIDTH({width}), .LO_TO_HI({1 if lo_to_hi else 0})) priority_encode_{out.net} (\n" + f" .in_value({src.net}), .index({index_net}), .valid({valid_net})\n" + f" );" + ) + assigns.append(f" assign {out.net} = {{{valid_net}, {index_net}}};") + continue + + if rhs.startswith("pyc.popcount "): + match = re.fullmatch( + r"pyc\.popcount\s+(.*?)\s*\{.*?\}\s*:\s*(\S+)\s*->\s*(\S+)", rhs + ) + if not match or len(lhs) != 1: + raise PYCVerilogError(f"cannot parse popcount: {line}") + inputs = _ssa_names(match.group(1)) + if len(inputs) != 1: + raise PYCVerilogError(f"popcount expects one operand: {line}") + src = _value(values, inputs[0]) + out = add_value(lhs[0], match.group(3)) + if out.width < src.width.bit_length(): + raise PYCVerilogError("popcount result width is too small") + instances.append( + f" pyc_popcount #(.IN_WIDTH({src.width}), .OUT_WIDTH({out.width})) popcount_{out.net} (\n" + f" .in({src.net}), .out({out.net})\n" + f" );" + ) + continue + + if rhs.startswith("pyc.divrem "): + match = re.fullmatch( + r"pyc\.divrem\s+(.*?)\s*:\s*\((.*?)\)\s*->\s*\((.*?)\)", + rhs, + ) + if not match or len(lhs) != 2: + raise PYCVerilogError(f"cannot parse divrem: {line}") + inputs = _ssa_names(match.group(1)) + annotated_inputs = _parse_types(match.group(2)) + annotated_outputs = _parse_types(match.group(3)) + if ( + len(inputs) != 4 + or annotated_inputs != ["i64", "i64", "i1", "i1"] + or annotated_outputs != ["i64", "i64"] + ): + raise PYCVerilogError( + "divrem expects i64 lhs/rhs, i1 signed/word, and two i64 results" + ) + lhs_value, rhs_value, signed_value, word_value = ( + _value(values, item) for item in inputs + ) + outputs = [add_value(lhs[0], "i64"), add_value(lhs[1], "i64")] + if [ + value.type for value in (lhs_value, rhs_value, signed_value, word_value) + ] != annotated_inputs: + raise PYCVerilogError("divrem operand types are inconsistent") + if divrem_adapter is not None: + raise PYCVerilogError( + "only one divrem transaction is supported per PYC module" + ) + if not fifo_payloads: + raise PYCVerilogError("divrem requires a surrounding queue FIFO") + + packet = fifo_payloads[-1] + packet_value = _value(values, packet["payload_name"]) + fifo_valid_value = _value(values, packet["out_valid_name"]) + fifo_ready_value = _value(values, packet["out_ready_name"]) + held_net = f"divrem_held_{packet_value.net}" + request_ready_net = f"divrem_request_ready_{outputs[0].net}" + response_valid_net = f"divrem_response_valid_{outputs[0].net}" + response_ready_net = f"divrem_response_ready_{outputs[0].net}" + declarations.extend( + [ + f" wire [{packet_value.width - 1}:0] {held_net};", + f" wire {request_ready_net};", + f" wire {response_valid_net};", + f" wire {response_ready_net};", + ] + ) + # Combinational values decoded from the input packet before the + # divrem op (opcode flags, operands, modes, and later selectors) + # must see the adapter's retained packet for the whole iterative + # transaction. packet_out is a pass-through before acceptance, + # so redirecting the direct packet reads is valid both at launch + # and while the response is pending. + packet_net_pattern = re.compile( + rf"(?\s*(\S+)", rhs) if not match or len(lhs) != 1: @@ -410,13 +655,55 @@ def add_value(name: str, type_: str) -> Value: ) continue - if rhs.startswith("pyc.select "): - match = re.fullmatch(r"pyc\.select\s+(.*?)\s*:\s*(.*?)\s*->\s*(\S+)", rhs) + if rhs.startswith("pyc.wrapping_bitfield "): + match = re.fullmatch( + r"pyc\.wrapping_bitfield\s+(.*?)\s*\{(.*?)\}\s*:\s*" + r"\((.*?)\)\s*->\s*(\S+)", + rhs, + ) if not match or len(lhs) != 1: - raise PYCVerilogError(f"cannot parse select: {line}") + raise PYCVerilogError(f"cannot parse wrapping bitfield: {line}") + inputs = _ssa_names(match.group(1)) + if len(inputs) != 5: + raise PYCVerilogError( + f"wrapping bitfield expects value, source, width, offset, mode: {line}" + ) + values_in = [_value(values, item) for item in inputs] + annotated_types = _parse_types(match.group(3)) + out = add_value(lhs[0], match.group(4)) + if ( + len(annotated_types) != 5 + or [value.type for value in values_in] != annotated_types + or [value.type for value in values_in] + != ["i64", "i64", "i7", "i7", "i4"] + or out.type != "i64" + ): + raise PYCVerilogError( + "wrapping bitfield operand/result types are inconsistent" + ) + semantic = re.search(r'\bsemantic_id\s*=\s*"([^"]+)"', match.group(2)) + if not semantic or semantic.group(1) != "pyc.wrapping_bitfield.v1": + raise PYCVerilogError("wrapping bitfield semantic_id is invalid") + instances.append( + " pyc_runtime_wrapping_bitfield #(.WIDTH(64), .CONTROL_WIDTH(7), .MODE_WIDTH(4)) " + f"wrapping_bitfield_{out.net} (\n" + f" .value({values_in[0].net}), .source({values_in[1].net}),\n" + f" .bit_width({values_in[2].net}), .bit_offset({values_in[3].net}),\n" + f" .mode({values_in[4].net}), .result({out.net})\n" + " );" + ) + continue + + if rhs.startswith("pyc.select ") or rhs.startswith("pyc.mux "): + op_name = "pyc.select" if rhs.startswith("pyc.select ") else "pyc.mux" + match = re.fullmatch( + rf"{re.escape(op_name)}\s+(.*?)\s*:\s*(.*?)\s*->\s*(\S+)", rhs + ) + if not match or len(lhs) != 1: + raise PYCVerilogError(f"cannot parse mux: {line}") inputs = _ssa_names(match.group(1)) if len(inputs) != 3: - raise PYCVerilogError(f"select expects condition, true, false: {line}") + raise PYCVerilogError(f"mux expects select, true, false: {line}") select, true_value, false_value = (_value(values, item) for item in inputs) annotated_types = _parse_types(match.group(2)) out = add_value(lhs[0], match.group(3)) @@ -426,7 +713,7 @@ def add_value(name: str, type_: str) -> Value: or true_value.type != out.type or false_value.type != out.type ): - raise PYCVerilogError("select condition/data types are inconsistent") + raise PYCVerilogError("mux select/data types are inconsistent") assigns.append( f" assign {out.net} = {select.net} ? {true_value.net} : {false_value.net};" ) @@ -442,7 +729,10 @@ def add_value(name: str, type_: str) -> Value: annotated_source = cast.group(3) out = add_value(lhs[0], cast.group(4)) if annotated_source is not None and source.type != annotated_source: - raise PYCVerilogError("cast source annotation does not match operand") + raise PYCVerilogError( + f"cast source annotation does not match operand: {line} " + f"(actual {source.type})" + ) if cast.group(1) == "alias" and source.type != out.type: raise PYCVerilogError("alias source/result types must match") if cast.group(1) == "trunc" and source.width < out.width: @@ -471,11 +761,65 @@ def add_value(name: str, type_: str) -> Value: source = _value(values, match.group(2)) if target.type != match.group(3) or source.type != target.type: raise PYCVerilogError("assign source/target types are inconsistent") - assigns.append(f" assign {target.net} = {source.net};") + if divrem_adapter and target.name == divrem_adapter["input_ready_name"]: + assigns.append( + f" assign {target.net} = {divrem_adapter['request_ready']};" + ) + else: + assigns.append(f" assign {target.net} = {source.net};") + continue + + shift_op = re.fullmatch( + r"pyc\.(shl|lshr|ashr)\s+(.*?)\s*:\s*(\S+)\s*,\s*(\S+)", + rhs, + ) + if shift_op and len(lhs) == 1: + inputs = _ssa_names(shift_op.group(2)) + if len(inputs) != 2: + raise PYCVerilogError(f"shift expects two operands: {line}") + left, right = (_value(values, item) for item in inputs) + if [left.type, right.type] != [shift_op.group(3), shift_op.group(4)]: + raise PYCVerilogError("shift operand annotations are inconsistent") + if left.type != right.type: + raise PYCVerilogError("shift amount type must match shifted value") + out = add_value(lhs[0], left.type) + operator = {"shl": "<<", "lshr": ">>", "ashr": ">>>"}[shift_op.group(1)] + expression = ( + f"$signed({left.net}) >>> {right.net}" + if shift_op.group(1) == "ashr" + else f"{left.net} {operator} {right.net}" + ) + assigns.append(f" assign {out.net} = {expression};") + continue + + comparison_op = re.fullmatch( + r'pyc\.cmp\s+(.*?)\s*\{\s*predicate\s*=\s*"(eq|ult|slt)"\s*\}' + r"\s*:\s*(.*?)\s*->\s*(\S+)", + rhs, + ) + if comparison_op and len(lhs) == 1: + inputs = _ssa_names(comparison_op.group(1)) + if len(inputs) != 2: + raise PYCVerilogError(f"cmp expects two operands: {line}") + left, right = (_value(values, item) for item in inputs) + operand_types = _parse_types(comparison_op.group(3)) + out = add_value(lhs[0], comparison_op.group(4)) + if operand_types != [left.type, right.type] or out.type != "i1": + raise PYCVerilogError("cmp operand/result types are inconsistent") + predicate = comparison_op.group(2) + if predicate == "slt": + expression = f"$signed({left.net}) < $signed({right.net})" + else: + expression = ( + f"{left.net} == {right.net}" + if predicate == "eq" + else f"{left.net} < {right.net}" + ) + assigns.append(f" assign {out.net} = {expression};") continue binary = re.fullmatch( - r"pyc\.(and|or|xor|add|sub|mul)\s+" + r"pyc\.(and|or|xor|add|sub|mul|shl|lshr|ashr|eq|ne|ult|slt|ule|ugt|uge)\s+" r"(.*?)\s*:\s*(.*?)\s*->\s*(\S+)", rhs, ) @@ -487,8 +831,19 @@ def add_value(name: str, type_: str) -> Value: operand_types = _parse_types(binary.group(3)) if operand_types != [left.type, right.type]: raise PYCVerilogError("binary operand annotations are inconsistent") + comparison = binary.group(1) in { + "eq", + "ne", + "ult", + "slt", + "ule", + "ugt", + "uge", + } out = add_value(lhs[0], binary.group(4)) - if out.type != left.type: + if comparison and out.type != "i1": + raise PYCVerilogError("comparison result must be i1") + if not comparison and out.type != left.type: raise PYCVerilogError("binary result type must match operands") operator = { "and": "&", @@ -497,33 +852,23 @@ def add_value(name: str, type_: str) -> Value: "add": "+", "sub": "-", "mul": "*", + "shl": "<<", + "lshr": ">>", + "ashr": ">>>", + "eq": "==", + "ne": "!=", + "ult": "<", + "slt": "<", + "ule": "<=", + "ugt": ">", + "uge": ">=", }[binary.group(1)] - expression = f"{left.net} {operator} {right.net}" - assigns.append(f" assign {out.net} = {expression};") - continue - - compare = re.fullmatch( - r'pyc\.cmp\s+(.*?)\s*\{\s*predicate\s*=\s*"(eq|ult|slt)"\s*\}' - r"\s*:\s*(.*?)\s*->\s*(\S+)", - rhs, - ) - if compare and len(lhs) == 1: - inputs = _ssa_names(compare.group(1)) - if len(inputs) != 2: - raise PYCVerilogError(f"cmp expects two operands: {line}") - left, right = (_value(values, item) for item in inputs) - if _parse_types(compare.group(3)) != [left.type, right.type]: - raise PYCVerilogError("cmp operand annotations are inconsistent") - out = add_value(lhs[0], compare.group(4)) - if out.type != "i1": - raise PYCVerilogError("cmp result must be i1") - predicate = compare.group(2) - if predicate == "eq": - expression = f"{left.net} == {right.net}" - elif predicate == "ult": - expression = f"{left.net} < {right.net}" - else: + if binary.group(1) == "slt": expression = f"$signed({left.net}) < $signed({right.net})" + elif binary.group(1) == "ashr": + expression = f"$signed({left.net}) >>> {right.net}" + else: + expression = f"{left.net} {operator} {right.net}" assigns.append(f" assign {out.net} = {expression};") continue @@ -571,7 +916,19 @@ def add_value(name: str, type_: str) -> Value: text.append("") text.extend(instances) text.extend(["", "endmodule", ""]) - text.extend(_runtime_sources(runtime_dir)) + body_text = "\n".join(module.body) + text.extend( + _runtime_sources( + runtime_dir, + include_fifo="pyc.fifo " in body_text, + include_reg="pyc.reg " in body_text, + include_popcount="pyc.popcount " in body_text, + include_rr_arbiter="pyc.rr_arbiter " in body_text, + include_wrapping_bitfield="pyc.wrapping_bitfield " in body_text, + include_divrem="pyc.divrem " in body_text, + include_priority="pyc.priority_encode " in body_text, + ) + ) text.extend(["", "/* verilator lint_on DECLFILENAME */", ""]) return "\n".join(text) diff --git a/compiler/mlir/include/pyc/Dialect/PYC/PYCOps.td b/compiler/mlir/include/pyc/Dialect/PYC/PYCOps.td index 585e310ec..e9d84db7a 100644 --- a/compiler/mlir/include/pyc/Dialect/PYC/PYCOps.td +++ b/compiler/mlir/include/pyc/Dialect/PYC/PYCOps.td @@ -88,6 +88,15 @@ def PYC_SremOp : PYC_Op<"srem", [Pure]> { let hasFolder = 1; } +def PYC_DivRemOp : PYC_Op<"divrem", [Pure]> { + let summary = "Shared signed/unsigned XLEN/word divide and remainder"; + let arguments = (ins AnyInteger:$lhs, AnyInteger:$rhs, I1:$signed_mode, + I1:$word_mode); + let results = (outs AnyInteger:$quotient, AnyInteger:$remainder); + let hasVerifier = 1; + let assemblyFormat = "$lhs `,` $rhs `,` $signed_mode `,` $word_mode attr-dict `:` `(` type($lhs) `,` type($rhs) `,` type($signed_mode) `,` type($word_mode) `)` `->` `(` type($quotient) `,` type($remainder) `)`"; +} + def PYC_SelectOp : PYC_Op<"select", [Pure]> { let summary = "Select one of two values (combinational)"; let arguments = (ins I1:$sel, AnyInteger:$a, AnyInteger:$b); @@ -239,6 +248,13 @@ def PYC_CountZerosOp : PYC_Op<"count_zeros", [Pure]> { let assemblyFormat = "$in attr-dict `:` type($in) `->` type($count)"; } +def PYC_WrappingBitfieldOp : PYC_Op<"wrapping_bitfield", [Pure]> { + let summary = "Dynamic wrapping bitfield semantic primitive"; + let arguments = (ins Variadic:$inputs, StrAttr:$semantic_id); + let results = (outs AnyInteger:$result); + let hasVerifier = 1; + let assemblyFormat = "$inputs attr-dict `:` `(` type($inputs) `)` `->` type($result)"; +} def PYC_RtlCombOp : PYC_Op<"rtl.comb", [Pure]> { let summary = "Selected qualified combinational RTL implementation"; let arguments = (ins Variadic:$inputs, diff --git a/compiler/mlir/lib/Dialect/PYC/PYCOps.cpp b/compiler/mlir/lib/Dialect/PYC/PYCOps.cpp index c7aff1a8d..166d9205a 100644 --- a/compiler/mlir/lib/Dialect/PYC/PYCOps.cpp +++ b/compiler/mlir/lib/Dialect/PYC/PYCOps.cpp @@ -393,9 +393,9 @@ OpFoldResult CmpOp::fold(FoldAdaptor adaptor) { auto a = asIntAttr(adaptor.getLhs()); auto b = asIntAttr(adaptor.getRhs()); if (a && b) { - bool value = predicate == "eq" ? (*a == *b) - : predicate == "ult" ? a->ult(*b) - : a->slt(*b); + bool value = predicate == "eq" + ? (*a == *b) + : predicate == "ult" ? a->ult(*b) : a->slt(*b); return IntegerAttr::get(IntegerType::get(getContext(), 1), value ? 1 : 0); } return {}; @@ -500,9 +500,9 @@ static OpFoldResult foldShift(Value input, Attribute inputAttr, } if (!value) return {}; - llvm::APInt result = kind == "shl" ? (*value << shift) - : kind == "lshr" ? value->lshr(shift) - : value->ashr(shift); + llvm::APInt result = + kind == "shl" ? (*value << shift) + : kind == "lshr" ? value->lshr(shift) : value->ashr(shift); return intAttrFor(resultType, result.trunc(outTy.getWidth())); } @@ -721,8 +721,7 @@ LogicalResult CountZerosOp::verify() { auto countType = dyn_cast(getCount().getType()); if (!inputType || !countType) return emitOpError("input and count result must be integer types"); - const auto *contract = - generated::findSemanticPrimitive("pyc.count_zeros.v1"); + const auto *contract = generated::findSemanticPrimitive("pyc.count_zeros.v1"); if (!contract) return emitOpError("semantic primitive is missing from the registry"); if (!generated::supportsInputWidth(*contract, inputType.getWidth())) @@ -753,6 +752,28 @@ static bool isSha256Fingerprint(llvm::StringRef value) { value, [](char c) { return llvm::isDigit(c) || (c >= 'a' && c <= 'f'); }); } +LogicalResult WrappingBitfieldOp::verify() { + auto semantic = (*this)->getAttrOfType("semantic_id"); + if (!semantic || semantic.getValue() != "pyc.wrapping_bitfield.v1") + return emitOpError("semantic_id must be pyc.wrapping_bitfield.v1"); + if (getInputs().size() != 5) + return emitOpError( + "requires value, source, width, offset, and mode inputs"); + auto valueTy = dyn_cast(getInputs()[0].getType()); + auto sourceTy = dyn_cast(getInputs()[1].getType()); + auto widthTy = dyn_cast(getInputs()[2].getType()); + auto offsetTy = dyn_cast(getInputs()[3].getType()); + auto modeTy = dyn_cast(getInputs()[4].getType()); + auto resultTy = dyn_cast(getResult().getType()); + if (!valueTy || !sourceTy || !widthTy || !offsetTy || !modeTy || !resultTy) + return emitOpError("all operands and result must be integer types"); + if (valueTy.getWidth() != 64 || sourceTy.getWidth() != 64 || + widthTy.getWidth() != 7 || offsetTy.getWidth() != 7 || + modeTy.getWidth() != 4 || resultTy.getWidth() != 64) + return emitOpError( + "requires i64 value/source/result, i7 width/offset, and i4 mode"); + return success(); +} LogicalResult RtlCombOp::verify() { if (getInputs().empty() || getOutputs().empty()) return emitOpError( @@ -1152,6 +1173,23 @@ DEFINE_VALUE_BINARY_VERIFY(XorOp) #undef DEFINE_VALUE_BINARY_VERIFY +LogicalResult DivRemOp::verify() { + auto lhs = dyn_cast(getLhs().getType()); + auto rhs = dyn_cast(getRhs().getType()); + auto quotient = dyn_cast(getQuotient().getType()); + auto remainder = dyn_cast(getRemainder().getType()); + auto signedMode = dyn_cast(getSignedMode().getType()); + auto wordMode = dyn_cast(getWordMode().getType()); + if (!lhs || !rhs || !quotient || !remainder || !signedMode || !wordMode) + return emitOpError("divrem operands and results must be integers"); + if (lhs != rhs || lhs.getWidth() != 64 || rhs.getWidth() != 64 || + quotient.getWidth() != 64 || remainder.getWidth() != 64 || + !signedMode.isSignless() || !wordMode.isSignless() || + signedMode.getWidth() != 1 || wordMode.getWidth() != 1) + return emitOpError("divrem requires i64 data and i1 mode operands"); + return success(); +} + LogicalResult CmpOp::verify() { StringRef predicate = getPredicate(); if (predicate != "eq" && predicate != "ult" && predicate != "slt") diff --git a/compiler/mlir/lib/Transforms/SelectRtlPrimitivesPass.cpp b/compiler/mlir/lib/Transforms/SelectRtlPrimitivesPass.cpp index 05b11f926..7764a0171 100644 --- a/compiler/mlir/lib/Transforms/SelectRtlPrimitivesPass.cpp +++ b/compiler/mlir/lib/Transforms/SelectRtlPrimitivesPass.cpp @@ -48,6 +48,37 @@ static std::string fingerprint(llvm::StringRef bytes) { return "sha256:" + llvm::toHex(hasher.final(), true); } +// Catalog digests describe Git's canonical text blobs. A Windows checkout +// may materialize those files with CRLF, so normalize only CRLF pairs before +// checking source and license content. +static std::string fingerprintRepositoryText(llvm::StringRef bytes) { + if (!bytes.contains("\r\n")) + return fingerprint(bytes); + std::string normalized; + normalized.reserve(bytes.size()); + for (size_t index = 0; index < bytes.size(); ++index) { + if (bytes[index] == '\r' && index + 1 < bytes.size() && + bytes[index + 1] == '\n') + continue; + normalized.push_back(bytes[index]); + } + return fingerprint(normalized); +} + +// These semantics are implementation details of wrapping_bitfield lowering, +// not public PYC operations. Keep the list closed here so catalog loading +// still rejects arbitrary semantic IDs while allowing the decomposed physical +// primitives to participate in selection. +static bool isBitfieldDecompositionSemantic(llvm::StringRef semanticId) { + return semanticId == "pyc.wrapping_field_normalize.v1" || + semanticId == "pyc.bitfield_clear.v1" || + semanticId == "pyc.bitfield_set.v1" || + semanticId == "pyc.bitfield_insert.v1" || + semanticId == "pyc.reverse_bytes.v1" || + semanticId == "pyc.dynamic_sign_extend.v1" || + semanticId == "pyc.runtime_zero_count.v1"; +} + static FailureOr> loadCatalog(llvm::StringRef path, std::string &catalogSha256, std::string &error) { @@ -96,30 +127,73 @@ loadCatalog(llvm::StringRef path, std::string &catalogSha256, auto licenseSha256 = entry ? entry->getString("license_sha256") : std::nullopt; bool knownSemanticShape = false; + auto hasPorts = [](const llvm::json::Array *actual, + std::initializer_list expected) { + if (!actual || actual->size() != expected.size()) + return false; + size_t index = 0; + for (llvm::StringRef name : expected) { + auto value = (*actual)[index++].getAsString(); + if (!value || *value != name) + return false; + } + return true; + }; + auto hasBindings = [](const llvm::json::Object *actual, + std::initializer_list expected) { + if (!actual || actual->size() != expected.size()) + return false; + return llvm::all_of( + expected, [&](llvm::StringRef name) { return actual->get(name); }); + }; if (semantic && inputPorts && outputPorts && bindings) { if (*semantic == "pyc.priority_encode.v1") - knownSemanticShape = inputPorts->size() == 1 && - inputPorts->front().getAsString() == "in_value" && - outputPorts->size() == 2 && - outputPorts->front().getAsString() == "index" && - (*outputPorts)[1].getAsString() == "valid" && - bindings->get("WIDTH") && - bindings->get("ORDER_LOW"); + knownSemanticShape = hasPorts(inputPorts, {"in_value"}) && + hasPorts(outputPorts, {"index", "valid"}) && + hasBindings(bindings, {"WIDTH", "ORDER_LOW"}); else if (*semantic == "pyc.popcount.v1") - knownSemanticShape = inputPorts->size() == 1 && - inputPorts->front().getAsString() == "in_value" && - outputPorts->size() == 1 && - outputPorts->front().getAsString() == "count" && - bindings->get("WIDTH") && - bindings->get("COUNT_WIDTH"); + knownSemanticShape = hasPorts(inputPorts, {"in_value"}) && + hasPorts(outputPorts, {"count"}) && + hasBindings(bindings, {"WIDTH", "COUNT_WIDTH"}); else if (*semantic == "pyc.count_zeros.v1") - knownSemanticShape = inputPorts->size() == 1 && - inputPorts->front().getAsString() == "in_value" && - outputPorts->size() == 1 && - outputPorts->front().getAsString() == "count" && - bindings->get("WIDTH") && - bindings->get("COUNT_WIDTH") && - bindings->get("DIRECTION_LOW"); + knownSemanticShape = + hasPorts(inputPorts, {"in_value"}) && + hasPorts(outputPorts, {"count"}) && + hasBindings(bindings, {"WIDTH", "COUNT_WIDTH", "DIRECTION_LOW"}); + else if (*semantic == "pyc.wrapping_bitfield.v1") + knownSemanticShape = + hasPorts(inputPorts, + {"value", "source", "bit_width", "bit_offset", "mode"}) && + hasPorts(outputPorts, {"result"}) && + hasBindings(bindings, {"WIDTH", "CONTROL_WIDTH", "MODE_WIDTH"}); + else if (*semantic == "pyc.wrapping_field_normalize.v1") + knownSemanticShape = + hasPorts(inputPorts, {"value", "bit_width", "bit_offset"}) && + hasPorts(outputPorts, {"field", "mask"}) && + hasBindings(bindings, {"WIDTH", "CONTROL_WIDTH"}); + else if (*semantic == "pyc.bitfield_clear.v1" || + *semantic == "pyc.bitfield_set.v1") + knownSemanticShape = hasPorts(inputPorts, {"value", "mask"}) && + hasPorts(outputPorts, {"result"}) && + hasBindings(bindings, {"WIDTH"}); + else if (*semantic == "pyc.bitfield_insert.v1") + knownSemanticShape = + hasPorts(inputPorts, + {"value", "source", "mask", "bit_width", "bit_offset"}) && + hasPorts(outputPorts, {"result"}) && + hasBindings(bindings, {"WIDTH", "CONTROL_WIDTH"}); + else if (*semantic == "pyc.reverse_bytes.v1") + knownSemanticShape = hasPorts(inputPorts, {"field", "bit_width"}) && + hasPorts(outputPorts, {"result"}) && + hasBindings(bindings, {"WIDTH", "CONTROL_WIDTH"}); + else if (*semantic == "pyc.dynamic_sign_extend.v1") + knownSemanticShape = hasPorts(inputPorts, {"field", "bit_width"}) && + hasPorts(outputPorts, {"result"}) && + hasBindings(bindings, {"WIDTH", "CONTROL_WIDTH"}); + else if (*semantic == "pyc.runtime_zero_count.v1") + knownSemanticShape = hasPorts(inputPorts, {"value", "direction_low"}) && + hasPorts(outputPorts, {"count"}) && + hasBindings(bindings, {"WIDTH", "COUNT_WIDTH"}); } if (!semantic || !implementation || effect != "comb" || !module || !minWidth || !maxWidth || !selectionPriority || *minWidth <= 0 || @@ -132,11 +206,13 @@ loadCatalog(llvm::StringRef path, std::string &catalogSha256, } const generated::SemanticPrimitiveContract *semanticContract = generated::findSemanticPrimitive(semantic->str()); - if (!semanticContract || - static_cast(*minWidth) < - semanticContract->minimumInputWidth || - static_cast(*maxWidth) > - semanticContract->maximumInputWidth) { + const bool isInternalBitfieldSemantic = + isBitfieldDecompositionSemantic(semantic->str()); + if ((!semanticContract && !isInternalBitfieldSemantic) || + (semanticContract && (static_cast(*minWidth) < + semanticContract->minimumInputWidth || + static_cast(*maxWidth) > + semanticContract->maximumInputWidth))) { error = "RTL primitive catalog entry is outside the semantic registry"; return failure(); } @@ -150,7 +226,8 @@ loadCatalog(llvm::StringRef path, std::string &catalogSha256, auto licenseBuffer = llvm::MemoryBuffer::getFile(licensePath); if (licenseFile->empty() || licenseFile->contains('\\') || licenseEscapes || !licenseBuffer || - fingerprint(licenseBuffer.get()->getBuffer()) != *licenseSha256) { + fingerprintRepositoryText(licenseBuffer.get()->getBuffer()) != + *licenseSha256) { error = "RTL primitive license file is missing or has a digest mismatch"; return failure(); } @@ -184,8 +261,8 @@ loadCatalog(llvm::StringRef path, std::string &catalogSha256, llvm::SmallString<256> sourceFile(llvm::sys::path::parent_path(path)); llvm::sys::path::append(sourceFile, *sourcePath); auto sourceBuffer = llvm::MemoryBuffer::getFile(sourceFile); - if (!sourceBuffer || - fingerprint(sourceBuffer.get()->getBuffer()) != *sourceSha) { + if (!sourceBuffer || fingerprintRepositoryText( + sourceBuffer.get()->getBuffer()) != *sourceSha) { error = "RTL primitive source digest mismatch for '" + sourcePath->str() + "'"; return failure(); @@ -244,10 +321,14 @@ struct SelectRtlPrimitivesPass SmallVector priorityOps; SmallVector popcountOps; SmallVector zeroCountOps; + SmallVector wrappingBitfieldOps; module.walk([&](PriorityEncodeOp op) { priorityOps.push_back(op); }); module.walk([&](PopcountOp op) { popcountOps.push_back(op); }); module.walk([&](CountZerosOp op) { zeroCountOps.push_back(op); }); - if (priorityOps.empty() && popcountOps.empty() && zeroCountOps.empty()) + module.walk( + [&](WrappingBitfieldOp op) { wrappingBitfieldOps.push_back(op); }); + if (priorityOps.empty() && popcountOps.empty() && zeroCountOps.empty() && + wrappingBitfieldOps.empty()) return; std::string path = catalog; @@ -316,15 +397,36 @@ struct SelectRtlPrimitivesPass return sourceValues; }; - for (PriorityEncodeOp semantic : priorityOps) { - unsigned width = cast(semantic.getIn().getType()).getWidth(); + auto createRtlComb = + [&](Operation *anchor, llvm::StringRef semanticId, + unsigned candidateWidth, ArrayRef operands, + ArrayRef resultTypes, ArrayRef parameters, + ArrayRef inputPorts, + ArrayRef outputPorts) -> Operation * { const RtlCandidate *candidate = - selectCandidate(semantic, "pyc.priority_encode.v1", width); - if (!candidate) { - signalPassFailure(); - return; - } + selectCandidate(anchor, semanticId, candidateWidth); + if (!candidate) + return nullptr; + OpBuilder builder(anchor); + OperationState state(anchor->getLoc(), RtlCombOp::getOperationName()); + state.addOperands(operands); + state.addTypes(resultTypes); + state.addAttribute("semantic_id", builder.getStringAttr(semanticId)); + state.addAttribute("implementation_id", + builder.getStringAttr(candidate->implementationId)); + state.addAttribute("module", builder.getStringAttr(candidate->module)); + state.addAttribute("parameters", builder.getDictionaryAttr(parameters)); + state.addAttribute("input_ports", builder.getStrArrayAttr(inputPorts)); + state.addAttribute("output_ports", builder.getStrArrayAttr(outputPorts)); + state.addAttribute("sources", builder.getArrayAttr( + sourceAttributes(builder, *candidate))); + state.addAttribute("catalog_sha256", + builder.getStringAttr(catalogSha256)); + return builder.create(state); + }; + for (PriorityEncodeOp semantic : priorityOps) { + unsigned width = cast(semantic.getIn().getType()).getWidth(); OpBuilder builder(semantic); SmallVector parameterValues{ builder.getNamedAttr( @@ -332,26 +434,15 @@ struct SelectRtlPrimitivesPass builder.getI64IntegerAttr(semantic.getOrder() == "low" ? 1 : 0)), builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width)), }; - SmallVector sourceValues = - sourceAttributes(builder, *candidate); - - OperationState state(semantic.getLoc(), RtlCombOp::getOperationName()); - state.addOperands(semantic.getIn()); - state.addTypes(semantic->getResultTypes()); - state.addAttribute("semantic_id", - builder.getStringAttr("pyc.priority_encode.v1")); - state.addAttribute("implementation_id", - builder.getStringAttr(candidate->implementationId)); - state.addAttribute("module", builder.getStringAttr(candidate->module)); - state.addAttribute("parameters", - builder.getDictionaryAttr(parameterValues)); - state.addAttribute("input_ports", builder.getStrArrayAttr({"in_value"})); - state.addAttribute("output_ports", - builder.getStrArrayAttr({"index", "valid"})); - state.addAttribute("sources", builder.getArrayAttr(sourceValues)); - state.addAttribute("catalog_sha256", - builder.getStringAttr(catalogSha256)); - Operation *selected = builder.create(state); + SmallVector inputs{semantic.getIn()}; + SmallVector outputs(semantic->getResultTypes()); + Operation *selected = createRtlComb( + semantic, "pyc.priority_encode.v1", width, inputs, outputs, + parameterValues, {"in_value"}, {"index", "valid"}); + if (!selected) { + signalPassFailure(); + return; + } semantic.getIndex().replaceAllUsesWith(selected->getResult(0)); semantic.getValid().replaceAllUsesWith(selected->getResult(1)); semantic.erase(); @@ -361,50 +452,281 @@ struct SelectRtlPrimitivesPass unsigned width = cast(semantic.getIn().getType()).getWidth(); unsigned countWidth = cast(semantic.getCount().getType()).getWidth(); - const RtlCandidate *candidate = - selectCandidate(semantic, "pyc.popcount.v1", width); - if (!candidate) { - signalPassFailure(); - return; - } OpBuilder builder(semantic); SmallVector parameterValues{ builder.getNamedAttr("COUNT_WIDTH", builder.getI64IntegerAttr(countWidth)), builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width)), }; - SmallVector sourceValues = - sourceAttributes(builder, *candidate); - OperationState state(semantic.getLoc(), RtlCombOp::getOperationName()); - state.addOperands(semantic.getIn()); - state.addTypes(semantic->getResultTypes()); - state.addAttribute("semantic_id", - builder.getStringAttr("pyc.popcount.v1")); - state.addAttribute("implementation_id", - builder.getStringAttr(candidate->implementationId)); - state.addAttribute("module", builder.getStringAttr(candidate->module)); - state.addAttribute("parameters", - builder.getDictionaryAttr(parameterValues)); - state.addAttribute("input_ports", builder.getStrArrayAttr({"in_value"})); - state.addAttribute("output_ports", builder.getStrArrayAttr({"count"})); - state.addAttribute("sources", builder.getArrayAttr(sourceValues)); - state.addAttribute("catalog_sha256", - builder.getStringAttr(catalogSha256)); - Operation *selected = builder.create(state); + SmallVector inputs{semantic.getIn()}; + SmallVector outputs(semantic->getResultTypes()); + Operation *selected = + createRtlComb(semantic, "pyc.popcount.v1", width, inputs, outputs, + parameterValues, {"in_value"}, {"count"}); + if (!selected) { + signalPassFailure(); + return; + } semantic.getCount().replaceAllUsesWith(selected->getResult(0)); semantic.erase(); } + for (WrappingBitfieldOp semantic : wrappingBitfieldOps) { + if (semantic.getInputs().size() != 5) { + semantic.emitError("wrapping_bitfield requires five inputs"); + signalPassFailure(); + return; + } + unsigned width = + cast(semantic.getInputs()[0].getType()).getWidth(); + unsigned controlWidth = + cast(semantic.getInputs()[2].getType()).getWidth(); + OpBuilder builder(semantic); + auto modeConstant = + semantic.getInputs()[4].getDefiningOp(); + if (!modeConstant) { + semantic.emitError( + "wrapping_bitfield mode must be constant so selection can use the " + "decomposed primitive path"); + signalPassFailure(); + return; + } + + uint64_t mode = modeConstant.getValueAttr().getValue().getZExtValue(); + if (mode > 8) { + semantic.emitError() + << "constant wrapping_bitfield mode must be in 0..8, got " << mode; + signalPassFailure(); + return; + } + + Value value = semantic.getInputs()[0]; + Value source = semantic.getInputs()[1]; + Value bitWidth = semantic.getInputs()[2]; + Value bitOffset = semantic.getInputs()[3]; + Type dataType = value.getType(); + SmallVector normalizeParameters{ + builder.getNamedAttr("CONTROL_WIDTH", + builder.getI64IntegerAttr(controlWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width)), + }; + SmallVector normalizeInputs{value, bitWidth, bitOffset}; + SmallVector normalizeOutputs{dataType, dataType}; + Operation *normalize = createRtlComb( + semantic, "pyc.wrapping_field_normalize.v1", width, normalizeInputs, + normalizeOutputs, normalizeParameters, + {"value", "bit_width", "bit_offset"}, {"field", "mask"}); + if (!normalize) { + signalPassFailure(); + return; + } + Value field = normalize->getResult(0); + Value mask = normalize->getResult(1); + Value replacement; + + auto selectSingleOutput = [&](llvm::StringRef semanticId, + ArrayRef inputs, + ArrayRef parameters, + ArrayRef inputPorts) { + SmallVector outputs{dataType}; + Operation *selected = + createRtlComb(semantic, semanticId, width, inputs, outputs, + parameters, inputPorts, {"result"}); + return selected ? selected->getResult(0) : Value(); + }; + + switch (mode) { + case 0: + replacement = field; + break; + case 1: { + SmallVector parameters{ + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{value, mask}; + replacement = selectSingleOutput("pyc.bitfield_clear.v1", inputs, + parameters, {"value", "mask"}); + break; + } + case 2: { + SmallVector parameters{ + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{value, mask}; + replacement = selectSingleOutput("pyc.bitfield_set.v1", inputs, + parameters, {"value", "mask"}); + break; + } + case 3: { + SmallVector parameters{ + builder.getNamedAttr("CONTROL_WIDTH", + builder.getI64IntegerAttr(controlWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{value, source, mask, bitWidth, bitOffset}; + replacement = selectSingleOutput( + "pyc.bitfield_insert.v1", inputs, parameters, + {"value", "source", "mask", "bit_width", "bit_offset"}); + break; + } + case 4: { + SmallVector parameters{ + builder.getNamedAttr("CONTROL_WIDTH", + builder.getI64IntegerAttr(controlWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{field, bitWidth}; + replacement = selectSingleOutput("pyc.reverse_bytes.v1", inputs, + parameters, {"field", "bit_width"}); + break; + } + case 5: { + SmallVector parameters{ + builder.getNamedAttr("CONTROL_WIDTH", + builder.getI64IntegerAttr(controlWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{field, bitWidth}; + replacement = selectSingleOutput("pyc.dynamic_sign_extend.v1", inputs, + parameters, {"field", "bit_width"}); + break; + } + case 6: { + unsigned countWidth = 1; + while ((uint64_t{1} << countWidth) < uint64_t{width} + 1) + ++countWidth; + Type countType = builder.getIntegerType(countWidth); + SmallVector parameters{ + builder.getNamedAttr("COUNT_WIDTH", + builder.getI64IntegerAttr(countWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{field}; + SmallVector outputs{countType}; + Operation *selected = + createRtlComb(semantic, "pyc.popcount.v1", width, inputs, outputs, + parameters, {"in_value"}, {"count"}); + if (selected) { + if (countType == dataType) + replacement = selected->getResult(0); + else + replacement = builder + .create(semantic.getLoc(), dataType, + selected->getResult(0)) + .getResult(); + } + break; + } + case 7: + case 8: { + unsigned countWidth = 1; + while ((uint64_t{1} << countWidth) < uint64_t{width} + 1) + ++countWidth; + unsigned effectiveWidthBits = std::max(controlWidth, countWidth); + auto effectiveType = builder.getIntegerType(effectiveWidthBits); + auto countType = builder.getIntegerType(countWidth); + Value widenedBitWidth = bitWidth; + if (controlWidth < effectiveWidthBits) + widenedBitWidth = builder + .create(semantic.getLoc(), + effectiveType, bitWidth) + .getResult(); + Value widthLimit = builder + .create( + semantic.getLoc(), effectiveType, + builder.getIntegerAttr(effectiveType, width)) + .getResult(); + Value widthIsOver = + builder + .create(semantic.getLoc(), builder.getI1Type(), + widthLimit, widenedBitWidth, + builder.getStringAttr("ult")) + .getResult(); + Value effectiveWidth = + builder + .create(semantic.getLoc(), effectiveType, + widthIsOver, widthLimit, widenedBitWidth) + .getResult(); + Value countInput = field; + if (mode == 7) { + Value zero = builder + .create( + semantic.getLoc(), effectiveType, + builder.getIntegerAttr(effectiveType, 0)) + .getResult(); + Value isZeroWidth = + builder + .create(semantic.getLoc(), builder.getI1Type(), + effectiveWidth, zero, + builder.getStringAttr("eq")) + .getResult(); + Value rawShift = + builder + .create(semantic.getLoc(), effectiveType, + widthLimit, effectiveWidth) + .getResult(); + Value safeShift = + builder + .create(semantic.getLoc(), effectiveType, + isZeroWidth, zero, rawShift) + .getResult(); + countInput = builder + .create(semantic.getLoc(), dataType, + field, safeShift) + .getResult(); + } + Value directionLow = + builder + .create( + semantic.getLoc(), builder.getI1Type(), + builder.getIntegerAttr(builder.getI1Type(), mode == 8)) + .getResult(); + SmallVector parameters{ + builder.getNamedAttr("COUNT_WIDTH", + builder.getI64IntegerAttr(countWidth)), + builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width))}; + SmallVector inputs{countInput, directionLow}; + SmallVector outputs{countType}; + Operation *selected = createRtlComb( + semantic, "pyc.runtime_zero_count.v1", width, inputs, outputs, + parameters, {"value", "direction_low"}, {"count"}); + if (!selected) + break; + Value rawCount = selected->getResult(0); + Value comparableCount = rawCount; + if (countWidth < effectiveWidthBits) + comparableCount = builder + .create(semantic.getLoc(), + effectiveType, rawCount) + .getResult(); + Value rawExceedsWidth = + builder + .create(semantic.getLoc(), builder.getI1Type(), + effectiveWidth, comparableCount, + builder.getStringAttr("ult")) + .getResult(); + Value clampedCount = + builder + .create(semantic.getLoc(), effectiveType, + rawExceedsWidth, effectiveWidth, + comparableCount) + .getResult(); + if (effectiveType == dataType) + replacement = clampedCount; + else + replacement = builder + .create(semantic.getLoc(), dataType, + clampedCount) + .getResult(); + break; + } + } + if (!replacement) { + signalPassFailure(); + return; + } + semantic.getResult().replaceAllUsesWith(replacement); + semantic.erase(); + } + for (CountZerosOp semantic : zeroCountOps) { unsigned width = cast(semantic.getIn().getType()).getWidth(); unsigned countWidth = cast(semantic.getCount().getType()).getWidth(); - const RtlCandidate *candidate = - selectCandidate(semantic, "pyc.count_zeros.v1", width); - if (!candidate) { - signalPassFailure(); - return; - } OpBuilder builder(semantic); SmallVector parameterValues{ builder.getNamedAttr("COUNT_WIDTH", @@ -415,24 +737,15 @@ struct SelectRtlPrimitivesPass semantic.getDirection() == "trailing" ? 1 : 0)), builder.getNamedAttr("WIDTH", builder.getI64IntegerAttr(width)), }; - SmallVector sourceValues = - sourceAttributes(builder, *candidate); - OperationState state(semantic.getLoc(), RtlCombOp::getOperationName()); - state.addOperands(semantic.getIn()); - state.addTypes(semantic->getResultTypes()); - state.addAttribute("semantic_id", - builder.getStringAttr("pyc.count_zeros.v1")); - state.addAttribute("implementation_id", - builder.getStringAttr(candidate->implementationId)); - state.addAttribute("module", builder.getStringAttr(candidate->module)); - state.addAttribute("parameters", - builder.getDictionaryAttr(parameterValues)); - state.addAttribute("input_ports", builder.getStrArrayAttr({"in_value"})); - state.addAttribute("output_ports", builder.getStrArrayAttr({"count"})); - state.addAttribute("sources", builder.getArrayAttr(sourceValues)); - state.addAttribute("catalog_sha256", - builder.getStringAttr(catalogSha256)); - Operation *selected = builder.create(state); + SmallVector inputs{semantic.getIn()}; + SmallVector outputs(semantic->getResultTypes()); + Operation *selected = + createRtlComb(semantic, "pyc.count_zeros.v1", width, inputs, outputs, + parameterValues, {"in_value"}, {"count"}); + if (!selected) { + signalPassFailure(); + return; + } semantic.getCount().replaceAllUsesWith(selected->getResult(0)); semantic.erase(); } diff --git a/compiler/mlir/tools/pycc.cpp b/compiler/mlir/tools/pycc.cpp index e79b1bd52..8a1024021 100644 --- a/compiler/mlir/tools/pycc.cpp +++ b/compiler/mlir/tools/pycc.cpp @@ -1057,6 +1057,20 @@ static std::string sha256Fingerprint(llvm::StringRef bytes) { return "sha256:" + llvm::toHex(hasher.final(), true); } +static std::string repositoryTextFingerprint(llvm::StringRef bytes) { + if (!bytes.contains("\r\n")) + return sha256Fingerprint(bytes); + std::string normalized; + normalized.reserve(bytes.size()); + for (size_t index = 0; index < bytes.size(); ++index) { + if (bytes[index] == '\r' && index + 1 < bytes.size() && + bytes[index + 1] == '\n') + continue; + normalized.push_back(bytes[index]); + } + return sha256Fingerprint(normalized); +} + static FailureOr> collectSelectedRtlSources(ModuleOp module) { std::vector result; @@ -1155,7 +1169,7 @@ static LogicalResult emitPrimitivesFile(llvm::StringRef outPath, return failure(); } llvm::StringRef content = fileOrErr->get()->getBuffer(); - if (sha256Fingerprint(content) != source.sha256) { + if (repositoryTextFingerprint(content) != source.sha256) { llvm::errs() << "error: selected RTL source digest mismatch: " << source.path << "\n"; return failure(); @@ -2436,6 +2450,15 @@ int main(int argc, char **argv) { pm.addPass(pyc::createCheckCombCyclesPass()); pm.addPass(pyc::createCheckClockDomainsPass()); pm.addNestedPass(pyc::createPackI1RegsPass()); + if (emitKind == "verilog") { + std::string catalogPath; + if (auto primitiveDir = findPrimitivesDir(argv[0])) { + llvm::SmallString<256> candidate(*primitiveDir); + llvm::sys::path::append(candidate, "rtl_catalog.json"); + catalogPath = candidate.str().str(); + } + pm.addPass(pyc::createSelectRtlPrimitivesPass(std::move(catalogPath))); + } const bool enableFuseComb = (!cppOnly) || !cppOnlyPreserveOps; if (enableFuseComb) pm.addNestedPass(pyc::createFuseCombPass()); @@ -2447,15 +2470,6 @@ int main(int argc, char **argv) { pm.addNestedPass(pyc::createCheckFlatTypesPass()); pm.addNestedPass(pyc::createCheckNoDynamicPass()); pm.addPass(pyc::createCheckLogicDepthPass(logicDepthLimit)); - if (emitKind == "verilog") { - std::string catalogPath; - if (auto primitiveDir = findPrimitivesDir(argv[0])) { - llvm::SmallString<256> candidate(*primitiveDir); - llvm::sys::path::append(candidate, "rtl_catalog.json"); - catalogPath = candidate.str().str(); - } - pm.addPass(pyc::createSelectRtlPrimitivesPass(std::move(catalogPath))); - } pm.addNestedPass(pyc::createCollectCompileStatsPass()); const auto tPassStart = Clock::now(); if (failed(pm.run(*module))) { diff --git a/docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json b/docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json new file mode 100644 index 000000000..7d20a75bb --- /dev/null +++ b/docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json @@ -0,0 +1,99 @@ +{ + "schema": "pyc-rtl-qualification-report-v1", + "run": "20260916-alu-bitfield-decomposition", + "status": "validated", + "scope": "DavinciOO IEX ALU constant-mode wrapping-bitfield decomposition", + "source": "designs/davincioo/spe/iex/alu.py", + "source_digests": { + "bitfield_primitives/pyc_wrapping_field_normalize.sv": "sha256:05216c60237a96a3bbd2f0b2920a10808fb84d4d6a9574f906692d034d8867d8", + "bitfield_primitives/pyc_bitfield_clear.sv": "sha256:faa64d52b0250eda3f2e6b7706f50ecda7607700f25867a89b06ebc1a8551d8e", + "bitfield_primitives/pyc_bitfield_set.sv": "sha256:4f9799028e14ec88692c1fe95239c537a81cadab39093edd5e7cb7220fdd2326", + "bitfield_primitives/pyc_bitfield_insert.sv": "sha256:44bdce1cd509eb20863d192ffc7794098644f340c235663fc211ae7c9ff294d6", + "bitfield_primitives/pyc_reverse_bytes.sv": "sha256:04ac0fe466de318b1b4c6af4c9a5fb8e202b4679497999acfe7e7afed1f6eb37", + "bitfield_primitives/pyc_dynamic_sign_extend.sv": "sha256:64a96ed7859f7c453cd1fb30600ce674d985ace0c46ee0669f4b1156a3571fc1", + "bitfield_primitives/pyc_popcount_primitive.sv": "sha256:019567028940095d7aa39d095078112ed15a45829dc3de792919426f8e46ef02", + "bitfield_primitives/pyc_runtime_zero_count.sv": "sha256:666bbc8290fe87b505eaa1604e1579186c5c9e830083069a25041e40d0c5eec8" + }, + "checks": { + "gfsim_e2e": { + "status": "passed", + "returncode": 0, + "command": "CXX=/usr/bin/clang++-22 /root/.pyenv/versions/3.11.15/bin/python3.11 -m pytest -q designs/davincioo/tests/spe/iex/test_alu.py", + "result": "2 passed", + "path": "alu.py -> ACIR -> QueueGraph -> generated gfsim C++ -> clang++-22/C++20 -> existing expected-result harness" + }, + "rtl_generation_e2e": { + "status": "passed", + "returncode": 0, + "commands": [ + "PYTHONPATH=python/semantic-core/src:python/pycircuit/src:python/agentic-circuit/src:. /root/.pyenv/versions/3.11.15/bin/python3.11 -c 'from pathlib import Path; import agentic_circuit as ac; from designs.davincioo.spe.iex.alu import alu_system; root=Path.cwd(); Path(\".pycircuit_out/davincioo-iex/alu-bitfield-20260916/raw.mlir\").write_text(ac.jit(alu_system, workspace=root).lower_acir(), encoding=\"utf-8\")'", + ".pycircuit_out/acir/dev-llvm22/bin/acir-opt --pass-pipeline='builtin.module(ac-lower-rules,canonicalize,cse,ac-verify-rule-closure,ac-freeze-topology)' .pycircuit_out/davincioo-iex/alu-bitfield-20260916/raw.mlir -o .pycircuit_out/davincioo-iex/alu-bitfield-20260916/frozen.mlir", + ".pycircuit_out/acir/dev-llvm22/bin/acir-queue-plan .pycircuit_out/davincioo-iex/alu-bitfield-20260916/frozen.mlir > .pycircuit_out/davincioo-iex/alu-bitfield-20260916/queuegraph.json", + ".pycircuit_out/acir/dev-llvm22/bin/acir-queue-pycgen .pycircuit_out/davincioo-iex/alu-bitfield-20260916/frozen.mlir > .pycircuit_out/davincioo-iex/alu-bitfield-20260916/alu.pyc", + "PYC_RTL_CATALOG=library/verilog/rtl_catalog.json PYC_PRIMITIVES_DIR=library/verilog .pycircuit_out/toolchain/build/bin/pycc .pycircuit_out/davincioo-iex/alu-bitfield-20260916/alu.pyc --emit=verilog --out-dir .pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl --hierarchy-policy=strict --inline-policy=off" + ], + "result": "regs=0, mems=0, max_depth=22/32, WNS=10, TNS=0" + }, + "verilator_functional": { + "status": "passed", + "returncode": 0, + "build_command": "CXX=clang++-22 /opt/oss-cad/oss-cad-suite/bin/verilator --binary --timing -Wall -Wno-fatal --compiler clang -CFLAGS '-std=gnu++20' -MAKEFLAGS 'CXX=clang++-22' --top-module alu_rtl_tb --Mdir .pycircuit_out/davincioo-iex/alu-bitfield-20260916/verilator-final2 .pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl/pyc_primitives.v .pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl/alu_system.v designs/davincioo/tests/spe/iex/alu_rtl_tb.sv", + "run_command": ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/verilator-final2/Valu_rtl_tb", + "result": "ALU_BITFIELD_RTL_PASS" + }, + "yosys_synthesis": { + "status": "passed", + "returncode": 0, + "command": "/opt/oss-cad/oss-cad-suite/bin/yosys -p 'read_verilog -sv pyc_primitives.v; read_verilog -sv alu_system.v; hierarchy -top alu_system; proc; opt; memory; opt; synth -noabc -top alu_system; check; stat'", + "working_directory": ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl", + "log": ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/yosys-noabc-final3.log", + "result": "End of script; Found and reported 0 problems" + }, + "catalog_unit": { + "status": "passed", + "command": "/root/.pyenv/versions/3.11.15/bin/python3.11 -m pytest -q tests/unit/test_primitive_catalog.py", + "result": "4 passed" + }, + "primitive_selection_system": { + "status": "passed", + "command": "PATH=/opt/oss-cad/oss-cad-suite/bin:/usr/lib/llvm-22/bin:/usr/bin:/bin CXX=/usr/bin/clang++-22 VERILATOR=/opt/oss-cad/oss-cad-suite/bin/verilator /root/.pyenv/versions/3.11.15/bin/python3.11 -m pytest -q tests/system/test_primitive_selection.py", + "result": "13 passed, 3 skipped" + } + }, + "directed_cases": { + "BXU": "passed, including wrapping bit_offset", + "BXS": "passed, sign bit 0 and 1", + "BCNT": "passed", + "BIC": "passed", + "BIS": "passed", + "BFI": "passed, including wrapping destination", + "REV": "passed for 8/16/32/64-bit fields and invalid width", + "CLZ": "passed for zero width, dynamic width below WIDTH, all-zero, and non-zero field", + "CTZ": "passed for zero width, dynamic width below WIDTH, all-zero, and non-zero field" + }, + "generated_hierarchy": { + "pyc_wrapping_field_normalize": 2, + "pyc_bitfield_clear": 1, + "pyc_bitfield_set": 1, + "pyc_bitfield_insert": 1, + "pyc_reverse_bytes": 1, + "pyc_dynamic_sign_extend": 2, + "pyc_popcount_primitive": 1, + "pyc_runtime_zero_count": 2, + "pyc_runtime_wrapping_bitfield": 0, + "pyc_count_zeros_primitive": 0, + "note": "CSE shares identical normalize results across constant-mode branches; each semantic WrappingBitfieldOp creates at most one normalize before optimization. CLZ and CTZ share the same physical zero-count module type with direction_low 0 and 1 respectively." + }, + "artifacts": [ + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/raw.mlir", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/frozen.mlir", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/queuegraph.json", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/alu.pyc", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl/manifest.json", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl/pyc_primitives.v", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/rtl/alu_system.v", + "designs/davincioo/tests/spe/iex/alu_rtl_tb.sv", + ".pycircuit_out/davincioo-iex/alu-bitfield-20260916/yosys-noabc-final3.log" + ] +} + diff --git a/library/verilog/bitfield_primitives/README.md b/library/verilog/bitfield_primitives/README.md new file mode 100644 index 000000000..8ee8f3cba --- /dev/null +++ b/library/verilog/bitfield_primitives/README.md @@ -0,0 +1,48 @@ +# Bitfield RTL Primitives + +This directory splits the previous monolithic wrapping-bitfield unit into +smaller composable RTL blocks. + +Recommended composition: + +```text +value + bit_width + bit_offset + | + v +pyc_wrapping_field_normalize + | | + field mask + | | + | +-----+------+ + | | | + | CLEAR SET + | + +--> dynamic_sign_extend (BXS / signed extract) + +--> popcount (BCNT) + +--> zero_count (CLZ / CTZ) + +--> reverse_bytes (REV) + +value + source + mask + width + offset + | + +--> bitfield_insert (BFI) +``` + +Files: + +- `pyc_wrapping_field_normalize.sv` +- `pyc_bitfield_clear.sv` +- `pyc_bitfield_set.sv` +- `pyc_bitfield_insert.sv` +- `pyc_reverse_bytes.sv` +- `pyc_dynamic_sign_extend.sv` +- `pyc_popcount_primitive.sv` +- `pyc_runtime_zero_count.sv` + +Notes: + +- `field` is packed into the low bits. +- `mask` marks selected positions in the original `value`. +- `pyc_runtime_zero_count` uses a runtime direction port so one physical + datapath can implement both CLZ and CTZ. +- These are low-level RTL building blocks. Compiler/PYC semantic operations + do not need to map 1:1 to every file. diff --git a/library/verilog/bitfield_primitives/pyc_bitfield_clear.sv b/library/verilog/bitfield_primitives/pyc_bitfield_clear.sv new file mode 100644 index 000000000..55af702cb --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_bitfield_clear.sv @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Clears the bit positions selected by mask. +module pyc_bitfield_clear #( + parameter integer WIDTH = 64 +) ( + input wire [WIDTH-1:0] value, + input wire [WIDTH-1:0] mask, + output wire [WIDTH-1:0] result +); + + assign result = value & ~mask; + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_bitfield_insert.sv b/library/verilog/bitfield_primitives/pyc_bitfield_insert.sv new file mode 100644 index 000000000..5e37b68ce --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_bitfield_insert.sv @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Inserts source[bit_width-1:0] into value starting at bit_offset. +// Destination positions wrap from bit WIDTH-1 back to bit 0. +// +// mask is normally produced by pyc_wrapping_field_normalize using the same +// bit_width and bit_offset. Passing it explicitly allows the normalization +// hardware to be shared by multiple downstream bitfield operations. +module pyc_bitfield_insert #( + parameter integer WIDTH = 64, + parameter integer CONTROL_WIDTH = (WIDTH <= 1) ? 1 : $clog2(WIDTH + 1) +) ( + input wire [WIDTH-1:0] value, + input wire [WIDTH-1:0] source, + input wire [WIDTH-1:0] mask, + input wire [CONTROL_WIDTH-1:0] bit_width, + input wire [CONTROL_WIDTH-1:0] bit_offset, + output reg [WIDTH-1:0] result +); + + integer i; + integer selected_width; + integer offset; + integer dst_index; + + always @(*) begin + result = value & ~mask; + + selected_width = bit_width; + offset = bit_offset; + dst_index = 0; + + if (selected_width > WIDTH) + selected_width = WIDTH; + + if (WIDTH > 0) begin + offset = offset % WIDTH; + + for (i = 0; i < WIDTH; i = i + 1) begin + if (i < selected_width) begin + dst_index = (offset + i) % WIDTH; + result[dst_index] = source[i]; + end + end + end + end + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_bitfield_set.sv b/library/verilog/bitfield_primitives/pyc_bitfield_set.sv new file mode 100644 index 000000000..220dc3fa0 --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_bitfield_set.sv @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Sets the bit positions selected by mask. +module pyc_bitfield_set #( + parameter integer WIDTH = 64 +) ( + input wire [WIDTH-1:0] value, + input wire [WIDTH-1:0] mask, + output wire [WIDTH-1:0] result +); + + assign result = value | mask; + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_dynamic_sign_extend.sv b/library/verilog/bitfield_primitives/pyc_dynamic_sign_extend.sv new file mode 100644 index 000000000..02cb35e56 --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_dynamic_sign_extend.sv @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Dynamically sign-extends a normalized field. +// +// field is expected to be packed into field[bit_width-1:0] by +// pyc_wrapping_field_normalize. +module pyc_dynamic_sign_extend #( + parameter integer WIDTH = 64, + parameter integer CONTROL_WIDTH = (WIDTH <= 1) ? 1 : $clog2(WIDTH + 1) +) ( + input wire [WIDTH-1:0] field, + input wire [CONTROL_WIDTH-1:0] bit_width, + output reg [WIDTH-1:0] result +); + + integer i; + integer selected_width; + reg sign_bit; + + always @(*) begin + result = {WIDTH{1'b0}}; + selected_width = bit_width; + sign_bit = 1'b0; + i = 0; + + if (selected_width > WIDTH) + selected_width = WIDTH; + + if (selected_width > 0) begin + sign_bit = field[selected_width - 1]; + + for (i = 0; i < WIDTH; i = i + 1) begin + if (i < selected_width) + result[i] = field[i]; + else + result[i] = sign_bit; + end + end + end + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_popcount_primitive.sv b/library/verilog/bitfield_primitives/pyc_popcount_primitive.sv new file mode 100644 index 000000000..f7c039512 --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_popcount_primitive.sv @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Canonical balanced-tree combinational population-count primitive. +module pyc_popcount_primitive #( + parameter integer WIDTH = 8, + parameter integer COUNT_WIDTH = 4 +) ( + input wire [WIDTH-1:0] in_value, + output wire [COUNT_WIDTH-1:0] count +); + localparam integer TREE_LEVELS = (WIDTH <= 1) ? 0 : $clog2(WIDTH); + localparam integer PAD_WIDTH = 1 << TREE_LEVELS; + + wire [COUNT_WIDTH-1:0] tree [0:(2 * PAD_WIDTH) - 2]; + + genvar node_index; + generate + for (node_index = 0; node_index < PAD_WIDTH - 1; node_index = node_index + 1) begin : gen_tree_nodes + assign tree[node_index] = tree[(2 * node_index) + 1] + tree[(2 * node_index) + 2]; + end + endgenerate + + genvar leaf_index; + generate + for (leaf_index = 0; leaf_index < PAD_WIDTH; leaf_index = leaf_index + 1) begin : gen_tree_leaves + if (leaf_index < WIDTH) begin : gen_input_leaf + assign tree[(PAD_WIDTH - 1) + leaf_index] = {{(COUNT_WIDTH-1){1'b0}}, in_value[leaf_index]}; + end else begin : gen_padding_leaf + assign tree[(PAD_WIDTH - 1) + leaf_index] = {COUNT_WIDTH{1'b0}}; + end + end + endgenerate + + assign count = tree[0]; +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_reverse_bytes.sv b/library/verilog/bitfield_primitives/pyc_reverse_bytes.sv new file mode 100644 index 000000000..112e60f6d --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_reverse_bytes.sv @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Reverses byte order inside a normalized field. +// +// field is expected to be packed into field[bit_width-1:0] by +// pyc_wrapping_field_normalize. For a legal REVERSE_BYTES operation, +// bit_width should be a non-zero multiple of 8. Other widths return zero. +module pyc_reverse_bytes #( + parameter integer WIDTH = 64, + parameter integer CONTROL_WIDTH = (WIDTH <= 1) ? 1 : $clog2(WIDTH + 1) +) ( + input wire [WIDTH-1:0] field, + input wire [CONTROL_WIDTH-1:0] bit_width, + output reg [WIDTH-1:0] result +); + + integer i; + integer selected_width; + integer byte_count; + integer src_byte; + integer dst_byte; + + always @(*) begin + result = {WIDTH{1'b0}}; + selected_width = bit_width; + i = 0; + byte_count = 0; + src_byte = 0; + dst_byte = 0; + + if (selected_width > WIDTH) + selected_width = WIDTH; + + if ((selected_width > 0) && ((selected_width % 8) == 0)) begin + byte_count = selected_width / 8; + + for (i = 0; i < WIDTH/8; i = i + 1) begin + if (i < byte_count) begin + dst_byte = i; + src_byte = byte_count - 1 - i; + result[(dst_byte * 8) +: 8] = field[(src_byte * 8) +: 8]; + end + end + end + end + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_runtime_zero_count.sv b/library/verilog/bitfield_primitives/pyc_runtime_zero_count.sv new file mode 100644 index 000000000..a1f1a3a9d --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_runtime_zero_count.sv @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Canonical combinational leading/trailing-zero count primitive. +// +// direction_low = 0: count from MSB toward LSB (CLZ/LZC) +// direction_low = 1: count from LSB toward MSB (CTZ) +// +// A single counting datapath is shared by CLZ and CTZ. +module pyc_runtime_zero_count #( + parameter integer WIDTH = 64, + parameter integer COUNT_WIDTH = (WIDTH <= 1) ? 1 : $clog2(WIDTH + 1) +) ( + input wire [WIDTH-1:0] value, + input wire direction_low, + output wire [COUNT_WIDTH-1:0] count +); + + wire [WIDTH-1:0] scan_value; + + genvar j; + generate + for (j = 0; j < WIDTH; j = j + 1) begin : gen_scan_order + // scan_value[0] is always the first bit inspected. + assign scan_value[j] = + direction_low ? value[j] : value[WIDTH - 1 - j]; + end + endgenerate + + reg [COUNT_WIDTH-1:0] count_reg; + reg found; + integer i; + + always @(*) begin + count_reg = {COUNT_WIDTH{1'b0}}; + found = 1'b0; + + for (i = 0; i < WIDTH; i = i + 1) begin + if (!found) begin + if (scan_value[i]) + found = 1'b1; + else + count_reg = count_reg + {{(COUNT_WIDTH-1){1'b0}}, 1'b1}; + end + end + end + + assign count = count_reg; + +endmodule diff --git a/library/verilog/bitfield_primitives/pyc_wrapping_field_normalize.sv b/library/verilog/bitfield_primitives/pyc_wrapping_field_normalize.sv new file mode 100644 index 000000000..c1ae199ae --- /dev/null +++ b/library/verilog/bitfield_primitives/pyc_wrapping_field_normalize.sv @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright (c) 2026 PTO-ISA +// +// Canonical dynamic wrapping-field normalization helper. +// +// Selects bit_width bits starting at bit_offset from value. Selection wraps +// from bit WIDTH-1 back to bit 0. The selected bits are packed into field +// starting at field[0]. mask marks the corresponding positions in value. +module pyc_wrapping_field_normalize #( + parameter integer WIDTH = 64, + parameter integer CONTROL_WIDTH = (WIDTH <= 1) ? 1 : $clog2(WIDTH + 1) +) ( + input wire [WIDTH-1:0] value, + input wire [CONTROL_WIDTH-1:0] bit_width, + input wire [CONTROL_WIDTH-1:0] bit_offset, + output reg [WIDTH-1:0] field, + output reg [WIDTH-1:0] mask +); + + integer i; + integer selected_width; + integer offset; + integer src_index; + + always @(*) begin + field = {WIDTH{1'b0}}; + mask = {WIDTH{1'b0}}; + + selected_width = bit_width; + offset = bit_offset; + src_index = 0; + + if (selected_width > WIDTH) + selected_width = WIDTH; + + if (WIDTH > 0) begin + offset = offset % WIDTH; + + for (i = 0; i < WIDTH; i = i + 1) begin + if (i < selected_width) begin + src_index = (offset + i) % WIDTH; + field[i] = value[src_index]; + mask[src_index] = 1'b1; + end + end + end + end + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/CHANGELOG.md b/library/verilog/div_pto_v2/div_pto_v2/CHANGELOG.md new file mode 100644 index 000000000..10df3a4f9 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/CHANGELOG.md @@ -0,0 +1,13 @@ +# Changes from supplied DIV candidate + +1. Added runtime `word_mode` so one 64-bit divider covers DIV/DIVU/DIVW/DIVUW + and REM/REMU/REMW/REMUW. +2. Added fixed-word operand normalization for signed and unsigned W forms. +3. Added mandatory sign-extension of every W-form quotient/remainder result. +4. Added PTO zero-divisor bypass: quotient=0, remainder=normalized dividend. +5. Added signed-minimum/-1 bypass: quotient=minimum, remainder=0. +6. Kept one physical BaseJump divider and dual quotient/remainder outputs. +7. Kept `BITS_PER_ITER` as the physical latency/area parameter. +8. Retained `divide_by_zero` only as a non-fault diagnostic output. +9. Added a directed RTL testbench and updated `files.f`. +10. Left all imported BaseJump RTL and its license unchanged. diff --git a/library/verilog/div_pto_v2/div_pto_v2/LICENSE.basejump b/library/verilog/div_pto_v2/div_pto_v2/LICENSE.basejump new file mode 100644 index 000000000..9af9041aa --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/LICENSE.basejump @@ -0,0 +1,12 @@ +This repository was created by Michael Taylor as educational/course +materials for the UCSD CSE 240B class. + +Copyright 2016 Michael B. Taylor. Copyright and related rights are licensed +under the Solderpad Hardware License, Version 0.51 (the “License”); you +may not use these files except in compliance with the License. You may +obtain a copy of the License at http://solderpad.org/licenses/SHL-0.51. +Unless required by applicable law or agreed to in writing, software, hardware +and materials distributed under this License is distributed on an “AS IS” +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied. See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/library/verilog/div_pto_v2/div_pto_v2/README.md b/library/verilog/div_pto_v2/div_pto_v2/README.md new file mode 100644 index 000000000..09199376c --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/README.md @@ -0,0 +1,90 @@ +# PTO scalar DIV/REM RTL primitive + +This directory contains a PTO-aware wrapper around the imported BaseJump +iterative integer divider. The BaseJump sources under `basejump/` are kept +unchanged; PTO architectural behavior is enforced at the wrapper boundary. + +## Supported operation mapping + +| PTO form | `is_signed` | `word_mode` | Architectural result selected by caller | +| --- | ---: | ---: | --- | +| DIV | 1 | 0 | quotient | +| DIVU | 0 | 0 | quotient | +| DIVW | 1 | 1 | quotient | +| DIVUW | 0 | 1 | quotient | +| REM | 1 | 0 | remainder | +| REMU | 0 | 0 | remainder | +| REMW | 1 | 1 | remainder | +| REMUW | 0 | 1 | remainder | + +The primitive computes quotient and remainder together. A higher layer selects +which output corresponds to the decoded mnemonic, so DIV and REM do not require +separate divider hardware. + +## PTO semantics implemented here + +The implementation follows `pto-spec/asl/scalar/model/alu/semantics.asl`: + +- DIV/DIVU/REM/REMU use full XLEN operands. +- Signed W forms sign-extend each low 32-bit operand before division. +- Unsigned W forms zero-extend each low 32-bit operand before division. +- **Every W-form result, including DIVUW/REMUW, sign-extends result[31:0] to XLEN.** +- A zero divisor is a total arithmetic case: quotient = 0, remainder = dividend + after the relevant operand normalization. +- Signed minimum divided by -1 is also total: quotient = signed minimum, + remainder = 0. + +Zero-divisor and signed-overflow cases bypass the iterative divider. This both +fixes the architectural result at the canonical boundary and avoids spending a +full iterative divide latency on a result already known from the ISA semantics. + +## New helper RTL + +- `pyc_word_operand_normalize.sv` + - full-width pass-through, signed low-word extension, or unsigned low-word + extension selected at runtime. + - This is **fixed-word normalization**, not wrapping-field normalization. +- `pyc_word_result_normalize.sv` + - sign-extends low 32 bits for every W-form result as required by PTO. +- `pyc_div_special_cases.sv` + - detects zero divisor and signed-minimum/-1 and generates raw quotient and + remainder without invoking the iterative divider. +- `pyc_runtime_div.sv` + - canonical ready/valid wrapper, special-case bypass, word-mode metadata + retention, and one BaseJump divider instance. + +## Files retained from the original package + +The `basejump/` sources and `LICENSE.basejump` are retained from the supplied +package. `BITS_PER_ITER` remains a structural parameter of the physical +implementation. Values 1 and 2 are the supported BaseJump configurations. + +## Directed testbench + +`tests/tb_pyc_runtime_div.sv` contains directed cases for: + +- normal signed and unsigned XLEN division/remainder, +- XLEN divide by zero, +- XLEN signed minimum / -1, +- DIVW signed behavior, +- DIVUW result sign extension, +- unsigned W zero-divisor remainder sign extension, +- signed W minimum / -1. + +Run the directed suite from this directory with: + +```sh +verilator --binary --timing --top-module tb_pyc_runtime_div \ + -Mdir -f tests/files_tb.f -Wno-fatal +/Vtb_pyc_runtime_div +``` + +The suite passes with Verilator 5.051. `pyc_runtime_div` and +`pyc_runtime_div_packet` also pass Yosys 0.68 hierarchy, process lowering, +optimization, and structural checks. + +## Interface note + +Compared with the original wrapper, the canonical interface adds `word_mode`. +The existing `divide_by_zero` output is retained only as an informational / +diagnostic signal; PTO does not treat divide-by-zero as an architectural fault. diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_adder_cin.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_adder_cin.sv new file mode 100644 index 000000000..0b67df9bd --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_adder_cin.sv @@ -0,0 +1,16 @@ +//This module implements a simple adder with cin +`include "bsg_defines.sv" + +module bsg_adder_cin #(parameter `BSG_INV_PARAM(width_p) + , harden_p=1) + ( input [width_p-1:0] a_i + , input [width_p-1:0] b_i + , input cin_i + , output [width_p-1:0] o + ); + + assign o = a_i + b_i + { {(width_p-1){1'b0}}, cin_i }; + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_adder_cin) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_counter_clear_up.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_counter_clear_up.sv new file mode 100644 index 000000000..205eefda8 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_counter_clear_up.sv @@ -0,0 +1,57 @@ +// This counter counts up and is occasionally cleared. +// If up and clear are applied on the same cycle, the +// clear occurs first, and then the up. +// + +`include "bsg_defines.sv" + +module bsg_counter_clear_up #(parameter `BSG_INV_PARAM(max_val_p) + // this originally had an "invalid" default value of -1 + // which is a bad choice for a counter + ,parameter init_val_p = `BSG_UNDEFINED_IN_SIM('0) + ,parameter ptr_width_lp = + `BSG_WIDTH(max_val_p) + ,parameter disable_overflow_warning_p = 0 + ) + (input clk_i + , input reset_i + + , input clear_i + , input up_i + // fixme: count_o should be renamed to count_r_o since some modules + // depend on this being a register and we want to indicate this at the interface level + , output logic [ptr_width_lp-1:0] count_o + ); + + // keeping track of number of entries and updating read and + // write pointers, and displaying errors in case of overflow + // or underflow + + always_ff @(posedge clk_i) + begin + if (reset_i) begin + count_o <= init_val_p; + end + else begin + if (clear_i) begin + count_o <= ptr_width_lp'(up_i); + end + else if (up_i) begin + count_o <= count_o + 1'b1; + end + end + end + +`ifndef BSG_HIDE_FROM_SYNTHESIS + + always_ff @ (negedge clk_i) + begin + if ((count_o==ptr_width_lp '(max_val_p)) && up_i && (reset_i===0) && !disable_overflow_warning_p) + $display("%m error: counter overflow at time %t", $time); + end + +`endif + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_counter_clear_up) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_defines.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_defines.sv new file mode 100644 index 000000000..d3d9a9d8b --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_defines.sv @@ -0,0 +1,219 @@ +`ifndef BSG_DEFINES_V +`define BSG_DEFINES_V + +`define BSG_MAX(x,y) (((x)>(y)) ? (x) : (y)) +`define BSG_MIN(x,y) (((x)<(y)) ? (x) : (y)) + +`define BSG_SIGN_EXTEND(sig, width) \ + ({{`BSG_MAX(width-$bits(sig),0){sig[$bits(sig)-1]}}, sig[0+:`BSG_MIN(width, $bits(sig))]}) +`define BSG_ZERO_EXTEND(sig, width) \ + ({{`BSG_MAX(width-$bits(sig),0){1'b0}}, sig[0+:`BSG_MIN(width, $bits(sig))]}) + +// place this macro at the end of a verilog module file if that module has invalid parameters +// that must be specified by the user. this will prevent that module from becoming a top-level +// module per the discussion here: https://github.com/SymbiFlow/sv-tests/issues/1160 and the +// SystemVerilog Standard + +// "Top-level modules are modules that are included in the SystemVerilog +// source text, but do not appear in any module instantiation statement, as +// described in 23.3.2. This applies even if the module instantiation appears +// in a generate block that is not itself instantiated (see 27.3). A design +// shall contain at least one top-level module. A top-level module is +// implicitly instantiated once, and its instance name is the same as the +// module name. Such an instance is called a top-level instance." +// + +`define BSG_ABSTRACT_MODULE(fn) \ + /*verilator lint_off DECLFILENAME*/ \ + /*verilator lint_off PINMISSING*/ \ + module fn``__abstract(); if (0) begin : abstract fn not_used(); end endmodule \ + /*verilator lint_on PINMISSING*/ \ + /*verilator lint_on DECLFILENAME*/ + +// macro for defining invalid parameter; with the abstract module declaration +// it should be sufficient to omit the "inv" but we include this for tool portability +// if later we find that all tools are compatible, we can remove the use of this from BaseJump STL + +`ifdef XCELIUM // Bare default parameters are incompatible as of 20.09.012 + // = "inv" causes type inference mismatch as of 20.09.012 +`define BSG_INV_PARAM(param) param = -1 +`elsif YOSYS // Use a bounded placeholder while Yosys reads parameterized modules. +`define BSG_INV_PARAM(param) param = 1 +`else // VIVADO, DC, VERILATOR, GENUS, SURELOG +`define BSG_INV_PARAM(param) param +`endif + + +// maps 1 --> 1 instead of to 0 +`define BSG_SAFE_CLOG2(x) ( (((x)==1) || ((x)==0))? 1 : $clog2((x))) +`define BSG_IS_POW2(x) ( (1 << $clog2(x)) == (x)) +`define BSG_WIDTH(x) ($clog2({1'b0, x}+1)) +`define BSG_SAFE_MINUS(x, y) (((x)<(y))) ? 0 : ((x)-(y)) + +// these "SAFE" shift functions handle a common problem that shifts default to 32 bits wide even +// though the result may be more bits than that. + +// these macros ensure that a left shift is done with sufficient bits of precision not to lose data +// extra is used if you want some extra bits of margin +`define BSG_SAFE_SHIFT_LEFT_BY_CONST(a,b_const,extra) (( ($bits(a) + (b_const) + extra )'(a)) << (b_const)) + +// b_max is the maximum value that b_variable can take on +`define BSG_SAFE_SHIFT_LEFT_CONST_BY_VARIABLE(a_const,b_variable,b_max,extra) (( ($clog2( (a_const)+1)+(b_max)+(extra)) ' (a_const)) << (b_variable) ) + + +// calculate ceil(x/y) +`define BSG_CDIV(x,y) (((x)+(y)-1)/(y)) + +`ifdef SYNTHESIS +`define BSG_UNDEFINED_IN_SIM(val) (val) +`else +`define BSG_UNDEFINED_IN_SIM(val) ('X) +`endif + +`ifdef VERILATOR +`define BSG_HIDE_FROM_VERILATOR(val) +`define BSG_VERILATOR_ONLY(val) val +`else +`define BSG_HIDE_FROM_VERILATOR(val) val +`define BSG_VERILATOR_ONLY(val) +`endif + +`ifdef SYNTHESIS +`define BSG_DISCONNECTED_IN_SIM(val) (val) +`elsif VERILATOR +`define BSG_DISCONNECTED_IN_SIM(val) (val) +`else +`define BSG_DISCONNECTED_IN_SIM(val) ('z) +`endif + +// Ufortunately per the Xilinx forums, Xilinx does not define +// any variable that indicates that Vivado Synthesis is running +// so as a result we identify Vivado merely as the exclusion of +// Synopsys Design Compiler (DC). Support beyond DC and Vivado +// will require modification of this macro. + +`ifdef SYNTHESIS + `ifdef DC + `define BSG_VIVADO_SYNTH_FAILS + `elsif CDS_TOOL_DEFINE + `define BSG_VIVADO_SYNTH_FAILS + `elsif SURELOG + `define BSG_VIVADO_SYNTH_FAILS + `elsif YOSYS + `define BSG_VIVADO_SYNTH_FAILS + `else + `define BSG_VIVADO_SYNTH_FAILS this_module_is_not_synthesizeable_in_vivado + `endif +`else +`define BSG_VIVADO_SYNTH_FAILS +`endif + +// macro for denoting that a code snippet is unsynthesiable + +`ifdef SYNTHESIS + `define BSG_HIDE_FROM_SYNTHESIS +`endif + +`ifdef SYNTHESIS +`define BSG_HIDE_FROM_SYNTHESIS_EXPR(val) +`else +`define BSG_HIDE_FROM_SYNTHESIS_EXPR(val) val +`endif + +`define BSG_STRINGIFY(x) `"x`" + + +// For the modules that must be hardened, add this macro at the top. +`ifdef SYNTHESIS +`define BSG_SYNTH_MUST_HARDEN this_module_must_be_hardened +`else +`define BSG_SYNTH_MUST_HARDEN +`endif + + +// using C-style shifts instead of a[i] allows the parameter of BSG_GET_BIT to be a parameter subrange +// e.g., parameter[4:1][1], which DC 2016.12 does not allow + +`define BSG_GET_BIT(X,NUM) (((X)>>(NUM))&1'b1) + +// This version of countones works in synthesis, but only up to 64 bits +// we do a funny thing where we propagate X's in simulation if it is more than 64 bits +// and in synthesis, go ahead and ignore the high bits + +`define BSG_COUNTONES_SYNTH(y) ((($bits(y) < 65) ? 1'b0 : `BSG_UNDEFINED_IN_SIM(1'b0)) + (`BSG_GET_BIT(y,0) +`BSG_GET_BIT(y,1) +`BSG_GET_BIT(y,2) +`BSG_GET_BIT(y,3) +`BSG_GET_BIT(y,4) +`BSG_GET_BIT(y,5) +`BSG_GET_BIT(y,6)+`BSG_GET_BIT(y,7) +`BSG_GET_BIT(y,8)+`BSG_GET_BIT(y,9) \ + +`BSG_GET_BIT(y,10)+`BSG_GET_BIT(y,11)+`BSG_GET_BIT(y,12)+`BSG_GET_BIT(y,13)+`BSG_GET_BIT(y,14)+`BSG_GET_BIT(y,15)+`BSG_GET_BIT(y,16)+`BSG_GET_BIT(y,17)+`BSG_GET_BIT(y,18)+`BSG_GET_BIT(y,19) \ + +`BSG_GET_BIT(y,20)+`BSG_GET_BIT(y,21)+`BSG_GET_BIT(y,22)+`BSG_GET_BIT(y,23)+`BSG_GET_BIT(y,24)+`BSG_GET_BIT(y,25)+`BSG_GET_BIT(y,26)+`BSG_GET_BIT(y,27)+`BSG_GET_BIT(y,28)+`BSG_GET_BIT(y,29) \ + +`BSG_GET_BIT(y,30)+`BSG_GET_BIT(y,31)+`BSG_GET_BIT(y,32)+`BSG_GET_BIT(y,33)+`BSG_GET_BIT(y,34)+`BSG_GET_BIT(y,35)+`BSG_GET_BIT(y,36)+`BSG_GET_BIT(y,37)+`BSG_GET_BIT(y,38)+`BSG_GET_BIT(y,39) \ + +`BSG_GET_BIT(y,40)+`BSG_GET_BIT(y,41)+`BSG_GET_BIT(y,42)+`BSG_GET_BIT(y,43)+`BSG_GET_BIT(y,44)+`BSG_GET_BIT(y,45)+`BSG_GET_BIT(y,46)+`BSG_GET_BIT(y,47)+`BSG_GET_BIT(y,48)+`BSG_GET_BIT(y,49) \ + +`BSG_GET_BIT(y,50)+`BSG_GET_BIT(y,51)+`BSG_GET_BIT(y,52)+`BSG_GET_BIT(y,53)+`BSG_GET_BIT(y,54)+`BSG_GET_BIT(y,55)+`BSG_GET_BIT(y,56)+`BSG_GET_BIT(y,57)+`BSG_GET_BIT(y,58)+`BSG_GET_BIT(y,59) \ + +`BSG_GET_BIT(y,60)+`BSG_GET_BIT(y,61)+`BSG_GET_BIT(y,62)+`BSG_GET_BIT(y,63))) + +// nullify rpgroups +`ifndef rpgroup +`define rpgroup(x) +`endif + +// verilog preprocessing -> if defined(A) && defined(B) then define C +`define BSG_DEFIF_A_AND_B(A,B,C) \ + `undef C \ + `ifdef A \ + `ifdef B \ + `define C \ + `endif \ + `endif + +// verilog preprocessing -> if defined(A) && !defined(B) then define C +`define BSG_DEFIF_A_AND_NOT_B(A,B,C) \ + `undef C \ + `ifdef A \ + `ifndef B \ + `define C \ + `endif \ + `endif + +// verilog preprocessing -> if !defined(A) && defined(B) then define C +`define BSG_DEFIF_NOT_A_AND_B(A,B,C) `BSG_DEFIF_A_AND_NOT_B(B,A,C) + +// verilog preprocessing -> if !defined(A) && !defined(B) then define C +`define BSG_DEFIF_NOT_A_AND_NOT_B(A,B,C) \ + `undef C \ + `ifndef A \ + `ifndef B \ + `define C \ + `endif \ + `endif + +// verilog preprocessing -> if defined(A) || defined(B) then define C +`define BSG_DEFIF_A_OR_B(A,B,C) \ + `undef C \ + `ifdef A \ + `define C \ + `endif \ + `ifdef B \ + `define C \ + `endif + +// verilog preprocessing -> if defined(A) || !defined(B) then define C +`define BSG_DEFIF_A_OR_NOT_B(A,B,C) \ + `undef C \ + `ifdef A \ + `define C \ + `endif \ + `ifndef B \ + `define C \ + `endif + +// verilog preprocessing -> if !defined(A) || defined(B) then define C +`define BSG_DEFIF_NOT_A_OR_B(A,B,C) `BSG_DEFIF_A_OR_NOT_B(B,A,C) + +// verilog preprocessing -> if !defined(A) || !defined(B) then define C +`define BSG_DEFIF_NOT_A_OR_NOT_B(A,B,C) \ + `undef C \ + `ifndef A \ + `define C \ + `endif \ + `ifndef B \ + `define C \ + `endif + +`endif diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_dff_en.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_dff_en.sv new file mode 100644 index 000000000..d7bed75ad --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_dff_en.sv @@ -0,0 +1,30 @@ +/** + * bsg_dff_en.sv + * @param width_p data width + */ + +`include "bsg_defines.sv" + +module bsg_dff_en #(parameter `BSG_INV_PARAM(width_p) + ,parameter harden_p=1 // mbt fixme: maybe this should not be a default + ,parameter strength_p=1) +( + input clk_i + ,input [width_p-1:0] data_i + ,input en_i + ,output logic [width_p-1:0] data_o +); + + logic [width_p-1:0] data_r; + + assign data_o = data_r; + + always_ff @ (posedge clk_i) begin + if (en_i) begin + data_r <= data_i; + end + end + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_dff_en) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative.sv new file mode 100644 index 000000000..d34fe8d2e --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative.sv @@ -0,0 +1,262 @@ +//==================================================================== +// bsg_idiv_iterative.sv +// 11/14/2016, shawnless.xie@gmail.com +//==================================================================== +// +// An N-bit integer iterative divider, capable of signed & unsigned division +// Code refactored based on Sam Larser's work +// ------------------------------------------- +// Cycles Operation +// ------------------------------------------- +// 1 latch inputs +// 2 negate divisor (if necessary) +// 3 negate dividend (if necessary) +// 4 shift in msb of the dividend +// 5-37 iterate +// 38 repair remainder (if necessary) +// 39 negate remainder (if necessary) +// 40 negate quotient (if necessary) +// ------------------------------------------- +// +// Schematic: https://docs.google.com/presentation/d/1F7Lam7fMCp-v9K1PsjTvypWHJFFXfqoX6pJrmgf-_JE/ +// +// TODO +// 1. added register to hold the previous operands, if the current operands +// are the same with prevous one, we can output the results instantly. This +// is useful for a RISC ISA, in which only quotient or remainder is need in +// one instruction. +// 2. usging data detection logic to reduce the iteration cycles. +`include "bsg_defines.sv" + +module bsg_idiv_iterative #(parameter width_p=32, parameter bitstack_p=0, parameter bits_per_iter_p = 1) + (input clk_i + ,input reset_i + + ,input v_i //there is a request + ,output ready_and_o //idiv is idle + + ,input [width_p-1: 0] dividend_i + ,input [width_p-1: 0] divisor_i + ,input signed_div_i + + ,output v_o //result is valid + ,output [width_p-1: 0] quotient_o + ,output [width_p-1: 0] remainder_o + ,input yumi_i + ); + +`ifndef BSG_HIDE_FROM_SYNTHESIS + initial begin + assert (bits_per_iter_p == 1 || bits_per_iter_p == 2) + else $error("Illegal value for parameters bits_per_iter_p given: (%0d). Legal values are 1 or 2.", bits_per_iter_p); + + assert (bits_per_iter_p == 1 || (bits_per_iter_p == 2 && (width_p % 2 == 0))) + else $error("Illegal value for parameter width_p: (%0d) given bits_per_iter_p: (%0d). Width must be even when resolving 2 bits per iteration", width_p, bits_per_iter_p); + end +`endif + + wire [width_p:0] opA_r; + assign remainder_o = opA_r[width_p-1:0]; + + wire [width_p:0] opC_r; + assign quotient_o = opC_r[width_p-1:0]; + + wire signed_div_r; + wire divisor_msb = signed_div_i & divisor_i[width_p-1]; + wire dividend_msb = signed_div_i & dividend_i[width_p-1]; + + wire latch_signed_div_lo; + bsg_dff_en#(.width_p(1)) req_reg + (.data_i (signed_div_i) + ,.data_o (signed_div_r) + ,.en_i (latch_signed_div_lo) + ,.clk_i(clk_i) + ); + + //if the divisor is zero + wire zero_divisor_li = ~(| opA_r); + + wire [1:0] opA_sel_lo; + wire [width_p:0] opA_mux; + + wire [width_p:0] add1_out, add2_out; + bsg_mux_one_hot #(.width_p(width_p+1), .els_p(2)) muxA + ( .data_i({ {divisor_msb, divisor_i}, add1_out } ) + ,.data_o( opA_mux ) + ,.sel_one_hot_i(opA_sel_lo) + ); + + wire [width_p:0] opB_mux, opC_mux; + wire [bits_per_iter_p + 1:0] opB_sel_lo, opC_sel_lo; + + if (bits_per_iter_p == 2) begin + + bsg_mux_one_hot #(.width_p(width_p+1), .els_p(4)) muxB + (.data_i( {opC_r, add1_out, {add1_out[width_p-1:0], opC_r[width_p]}, {add2_out[width_p-1:0], opC_r[width_p-1]}} ) + ,.data_o( opB_mux ) + ,.sel_one_hot_i(opB_sel_lo) + ); + + bsg_mux_one_hot #(.width_p(width_p+1), .els_p(4)) muxC + (.data_i( {{dividend_msb, dividend_i},add1_out, {opC_r[width_p-1:0], ~add1_out[width_p]}, {opC_r[width_p-2:0], ~add1_out[width_p], ~add2_out[width_p]}}) + ,.data_o( opC_mux ) + ,.sel_one_hot_i(opC_sel_lo) + ); + + end else begin + + bsg_mux_one_hot #(.width_p(width_p+1), .els_p(3)) muxB + (.data_i( {opC_r, add1_out, {add1_out[width_p-1:0], opC_r[width_p]}} ) + ,.data_o( opB_mux ) + ,.sel_one_hot_i(opB_sel_lo) + ); + + bsg_mux_one_hot #(.width_p(width_p+1), .els_p(3)) muxC + (.data_i( {{dividend_msb, dividend_i},add1_out, {opC_r[width_p-1:0], ~add1_out[width_p]}} ) + ,.data_o( opC_mux ) + ,.sel_one_hot_i(opC_sel_lo) + ); + + end + + wire opA_ld_lo; + bsg_dff_en#(.width_p(width_p+1)) opA_reg + (.data_i (opA_mux) + ,.data_o (opA_r ) + ,.en_i (opA_ld_lo ) + ,.clk_i(clk_i) + ); + + wire opB_ld_lo; + wire [width_p:0] opB_r; + bsg_dff_en#(.width_p(width_p+1)) opB_reg + (.data_i (opB_mux) + ,.data_o (opB_r ) + ,.en_i (opB_ld_lo ) + ,.clk_i(clk_i) + ); + + wire opC_ld_lo; + bsg_dff_en#(.width_p(width_p+1)) opC_reg + (.data_i (opC_mux) + ,.data_o (opC_r ) + ,.en_i (opC_ld_lo ) + ,.clk_i(clk_i) + ); + + wire opA_inv_lo; + wire opB_inv_lo; + wire opA_clr_lo; + wire opB_clr_lo; + + wire [width_p:0] add1_in0; + wire [width_p:0] add1_in1; + wire [width_p:0] add2_in0; + wire [width_p:0] add2_in1; + + // this logic is sandwiched between bitstacks -- MBT + if (bitstack_p) begin: bs + + wire [width_p:0] opA_xnor; + bsg_xnor#(.width_p(width_p+1)) xnor_opA + (.a_i({(width_p+1){opA_inv_lo}}) + ,.b_i(opA_r) + ,.o (opA_xnor) + ); + + wire [width_p:0] opB_xnor; + bsg_xnor#(.width_p(width_p+1)) xnor_opB + (.a_i({(width_p+1){opB_inv_lo}}) + ,.b_i(opB_r) + ,.o (opB_xnor) + ); + + bsg_nor2 #(.width_p(width_p+1)) nor_opA + ( .a_i( opA_xnor ) + ,.b_i({(width_p+1){~opA_clr_lo}}) + ,.o (add1_in0) + ); + + bsg_nor2 #(.width_p(width_p+1)) nor_opB + ( .a_i( opB_xnor ) + ,.b_i( {(width_p+1){~opB_clr_lo}}) + ,.o (add1_in1) + ); + + if (bits_per_iter_p == 2) begin + bsg_xor#(.width_p(width_p+1)) xor_add1 + (.a_i({(width_p+1){~add1_out[width_p]}}) + ,.b_i(opA_r) + ,.o (add2_in0) + ); + assign add2_in1 = {add1_out[width_p-1:0], opC_r[width_p]}; + end + + end + else begin: nbs + assign add1_in0 = (opA_r ^ {width_p+1{opA_inv_lo}}) & {width_p+1{opA_clr_lo}}; + assign add1_in1 = (opB_r ^ {width_p+1{opB_inv_lo}}) & {width_p+1{opB_clr_lo}}; + + if (bits_per_iter_p == 2) begin + assign add2_in0 = (opA_r ^ {width_p+1{~add1_out[width_p]}}); + assign add2_in1 = {add1_out[width_p-1:0], opC_r[width_p]}; + end + + end + + + wire adder1_cin_lo; + bsg_adder_cin #(.width_p(width_p+1)) adder1 + (.a_i (add1_in0) + ,.b_i (add1_in1) + ,.cin_i(adder1_cin_lo) + ,.o (add1_out) + ); + + if (bits_per_iter_p == 2) begin + wire adder2_cin = ~add1_out[width_p]; + bsg_adder_cin #(.width_p(width_p+1)) adder2 + (.a_i (add2_in0) + ,.b_i (add2_in1) + ,.cin_i(adder2_cin) + ,.o (add2_out) + ); + end + else begin + assign add2_out = '0; + end + + bsg_idiv_iterative_controller #(.width_p(width_p), .bits_per_iter_p(bits_per_iter_p)) control + ( .reset_i (reset_i) + ,.clk_i (clk_i) + + ,.v_i (v_i) + ,.ready_and_o (ready_and_o) + + ,.zero_divisor_i (zero_divisor_li) + ,.signed_div_r_i (signed_div_r) + ,.adder1_result_is_neg_i (add1_out[width_p]) + ,.adder2_result_is_neg_i (add2_out[width_p]) + ,.opA_is_neg_i (opA_r[width_p]) + ,.opC_is_neg_i (opC_r[width_p]) + + ,.opA_sel_o (opA_sel_lo) + ,.opA_ld_o (opA_ld_lo) + ,.opA_inv_o (opA_inv_lo) + ,.opA_clr_l_o (opA_clr_lo) + + ,.opB_sel_o (opB_sel_lo) + ,.opB_ld_o (opB_ld_lo) + ,.opB_inv_o (opB_inv_lo) + ,.opB_clr_l_o (opB_clr_lo) + + ,.opC_sel_o (opC_sel_lo) + ,.opC_ld_o (opC_ld_lo) + + ,.latch_signed_div_o (latch_signed_div_lo) + ,.adder1_cin_o (adder1_cin_lo) + + ,.v_o(v_o) + ,.yumi_i(yumi_i) + ); +endmodule // divide diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative_controller.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative_controller.sv new file mode 100644 index 000000000..ad0e04569 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_idiv_iterative_controller.sv @@ -0,0 +1,267 @@ +//==================================================================== +// bsg_idiv_iterative_controller.sv +// 11/14/2016, shawnless.xie@gmail.com +//==================================================================== +// +// The controller of bsg_idiv_iterative module. +// Code refactored based on Sam Larser's work + +`include "bsg_defines.sv" + +module bsg_idiv_iterative_controller #(parameter width_p=32, parameter bits_per_iter_p = 1) + (input clk_i + ,input reset_i + + ,input v_i + ,output ready_and_o + + ,input zero_divisor_i + ,input signed_div_r_i + ,input adder1_result_is_neg_i + ,input adder2_result_is_neg_i + ,input opA_is_neg_i + ,input opC_is_neg_i + + ,output logic [1:0] opA_sel_o + ,output logic opA_ld_o + ,output logic opA_inv_o + ,output logic opA_clr_l_o + + ,output logic [bits_per_iter_p + 1:0] opB_sel_o + ,output logic opB_ld_o + ,output logic opB_inv_o + ,output logic opB_clr_l_o + + ,output logic [bits_per_iter_p + 1:0] opC_sel_o + ,output logic opC_ld_o + + ,output logic latch_signed_div_o + ,output logic adder1_cin_o + + ,output logic v_o + ,input yumi_i + ); + + logic q_neg_r; + logic r_neg_r; + logic neg_ld; + logic add1_neg_last_r, add2_neg_last_r; + + typedef enum logic[5:0] + {WAIT, NEG0, NEG1, SHIFT, + CALC, + REPAIR, REMAIN, + QUOT,DONE } idiv_ctrl_stat; + idiv_ctrl_stat state, next_state; + + always @(posedge clk_i) begin + add1_neg_last_r <= adder1_result_is_neg_i; + add2_neg_last_r <= adder2_result_is_neg_i; + + if (neg_ld) begin + // the quotient is negated if the signs of the operands differ + q_neg_r <= (opA_is_neg_i ^ opC_is_neg_i) & signed_div_r_i; + + // the remainder is negated if the dividend is negative + r_neg_r <= opC_is_neg_i & signed_div_r_i; + end + end + + logic [`BSG_WIDTH(width_p/bits_per_iter_p)-1:0] calc_cnt; + wire calc_up_li = (state == CALC) && (calc_cnt < width_p/bits_per_iter_p); + wire calc_done = (calc_cnt == width_p/bits_per_iter_p); + bsg_counter_clear_up#(.max_val_p(width_p/bits_per_iter_p) + ,.init_val_p(0) + ,.disable_overflow_warning_p(1)) calc_counter + (.clk_i(clk_i) + ,.reset_i(reset_i) + + // We rely on natural overflow + ,.clear_i(calc_done) + ,.up_i(calc_up_li) + + ,.count_o(calc_cnt) + ); + +// synopsys sync_set_reset "reset_i" + always @(posedge clk_i) begin + if (reset_i) state <= WAIT; + else state <= next_state; + end + + always_comb begin + opA_sel_o = 2'b00; + opA_ld_o = 1'b0; + + if (bits_per_iter_p == 2) + opA_inv_o = !add2_neg_last_r; + else + opA_inv_o = !add1_neg_last_r; + + opA_clr_l_o = 1'b1; + + if (bits_per_iter_p == 2) + opB_sel_o = 4'b0001; + else + opB_sel_o = 3'b001; + + opB_ld_o = 1'b0; + opB_inv_o = 1'b0; + opB_clr_l_o = 1'b1; + opC_sel_o = 3'b001; + opC_ld_o = 1'b0; + + if (bits_per_iter_p == 2) + adder1_cin_o = !add2_neg_last_r; + else + adder1_cin_o = !add1_neg_last_r; + + neg_ld = 1'b0; + latch_signed_div_o = 1'b0; + next_state = WAIT; + + case (state) + + WAIT: begin + if (v_i) begin + opA_ld_o = 1'b1; + opC_ld_o = 1'b1; + latch_signed_div_o = 1'b1; + next_state = NEG0; + opA_sel_o = 2'b10; + + if (bits_per_iter_p == 2) + opC_sel_o = 4'b1000; + else + opC_sel_o = 3'b100; + end + + end + + NEG0: begin + next_state = (opC_is_neg_i & signed_div_r_i) ? NEG1 : SHIFT; + opA_sel_o = 2'b01; + opA_inv_o = 1'b1; + opB_clr_l_o = 1'b0; + opB_ld_o = 1'b1; + opC_ld_o = 1'b0; + neg_ld = 1'b1; + adder1_cin_o = 1'b1; + opA_ld_o = opA_is_neg_i & signed_div_r_i; + + if (bits_per_iter_p == 2) + opB_sel_o = 4'b1000; + else + opB_sel_o = 3'b100; + end + + NEG1: begin + next_state = SHIFT; + opA_clr_l_o = 1'b0; + opB_inv_o = 1'b1; + opB_ld_o = 1'b0; + opC_ld_o = 1'b1; + adder1_cin_o = 1'b1; + + if (bits_per_iter_p == 2) + opC_sel_o = 4'b0100; + else + opC_sel_o = 3'b010; + end + + SHIFT: begin + next_state = CALC; + opB_ld_o = 1'b1; + opC_ld_o = 1'b1; + opA_clr_l_o = 1'b0; + opB_clr_l_o = 1'b0; + adder1_cin_o = 1'b0; + + if (bits_per_iter_p == 2) begin + opC_sel_o = 4'b0010; + opB_sel_o = 4'b0010; + end else begin + opC_sel_o = 3'b001; + opB_sel_o = 3'b001; + end + end + + CALC: begin + opB_ld_o = 1'b1; + opC_ld_o = 1'b1; + if (bits_per_iter_p == 2) begin + opB_sel_o = calc_done ? 4'b0100 : 4'b0001; + opC_sel_o = calc_done ? 4'b0010 : 4'b0001; + if (calc_cnt == 0) begin + opA_inv_o = !add1_neg_last_r; + adder1_cin_o = !add1_neg_last_r; + end + end else + opB_sel_o = calc_done ? 3'b010 : 3'b001; + + if (calc_done) begin + if (adder1_result_is_neg_i) next_state = REPAIR; + else next_state = REMAIN; + end else + next_state = CALC; + end + + REPAIR: begin + next_state = REMAIN; + opA_inv_o = 1'b0; + opB_ld_o = 1'b1; + opC_ld_o = 1'b0; + adder1_cin_o = 1'b0; + + if (bits_per_iter_p == 2) + opB_sel_o = 4'b0100; + else + opB_sel_o = 3'b010; + end + + REMAIN: begin + next_state = (zero_divisor_i | !q_neg_r) ? DONE: QUOT; + opA_sel_o = 2'b01; + opA_ld_o = 1'b1; + opA_clr_l_o = 1'b0; + opB_ld_o = 1'b1; + opC_ld_o = 1'b0; + opB_inv_o = r_neg_r; + adder1_cin_o = r_neg_r; + + if (bits_per_iter_p == 2) + opB_sel_o = 4'b1000; + else + opB_sel_o = 3'b100; + end + + QUOT: begin + next_state = DONE; + opA_clr_l_o = 1'b0; + opB_inv_o = 1'b1; + opB_ld_o = 1'b0; + opC_ld_o = 1'b1; + adder1_cin_o = 1'b1; + + if (bits_per_iter_p == 2) + opC_sel_o = 4'b0100; + else + opC_sel_o = 3'b010; + end + + DONE:begin + if( yumi_i ) next_state = WAIT; + else next_state = DONE; + + opA_ld_o = 1'b0; + opB_ld_o = 1'b0; + opC_ld_o = 1'b0; + end + + endcase + end + + assign ready_and_o = ( state == WAIT ); + assign v_o = ( state == DONE ); + +endmodule // divide_control diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_mux_one_hot.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_mux_one_hot.sv new file mode 100644 index 000000000..c0c249d72 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_mux_one_hot.sv @@ -0,0 +1,42 @@ + +`include "bsg_defines.sv" + +module bsg_mux_one_hot #(parameter `BSG_INV_PARAM(width_p) + , parameter integer els_p=1 + , parameter integer harden_p=1 + ) + ( + input [els_p-1:0][width_p-1:0] data_i + ,input [els_p-1:0] sel_one_hot_i + ,output [width_p-1:0] data_o + ); + + wire [els_p-1:0][width_p-1:0] data_masked; + + genvar i,j; + + for (i = 0; i < els_p; i++) + begin : mask + assign data_masked[i] = data_i[i] & { width_p { sel_one_hot_i[i] } }; + end + + for (i = 0; i < width_p; i++) + begin: reduce + wire [els_p-1:0] gather; + + for (j = 0; j < els_p; j++) + begin : reduce2 + assign gather[j] = data_masked[j][i]; + end + + assign data_o[i] = | gather; + end + + if (els_p == 0) + begin : zero + assign data_o = '0; + end + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_mux_one_hot) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_nor2.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_nor2.sv new file mode 100644 index 000000000..eedbd612d --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_nor2.sv @@ -0,0 +1,14 @@ +`include "bsg_defines.sv" + +module bsg_nor2 #(parameter `BSG_INV_PARAM(width_p) + , harden_p=1) + (input [width_p-1:0] a_i + , input [width_p-1:0] b_i + , output [width_p-1:0] o + ); + + assign o = ~(a_i | b_i ); + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_nor2) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xnor.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xnor.sv new file mode 100644 index 000000000..e80bc4084 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xnor.sv @@ -0,0 +1,14 @@ +`include "bsg_defines.sv" + +module bsg_xnor #(parameter `BSG_INV_PARAM(width_p) + , harden_p=1) + (input [width_p-1:0] a_i + , input [width_p-1:0] b_i + , output [width_p-1:0] o + ); + + assign o = ~(a_i ^ b_i); + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_xnor) diff --git a/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xor.sv b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xor.sv new file mode 100644 index 000000000..5fb1b0d09 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/basejump/bsg_xor.sv @@ -0,0 +1,14 @@ +`include "bsg_defines.sv" + +module bsg_xor #(parameter `BSG_INV_PARAM(width_p) + , harden_p=1) + (input [width_p-1:0] a_i + , input [width_p-1:0] b_i + , output [width_p-1:0] o + ); + + assign o = a_i ^ b_i; + +endmodule + +`BSG_ABSTRACT_MODULE(bsg_xor) diff --git a/library/verilog/div_pto_v2/div_pto_v2/files.f b/library/verilog/div_pto_v2/div_pto_v2/files.f new file mode 100644 index 000000000..c9ba839c6 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/files.f @@ -0,0 +1,16 @@ ++incdir+basejump +basejump/bsg_defines.sv +basejump/bsg_dff_en.sv +basejump/bsg_mux_one_hot.sv +basejump/bsg_xnor.sv +basejump/bsg_nor2.sv +basejump/bsg_xor.sv +basejump/bsg_adder_cin.sv +basejump/bsg_counter_clear_up.sv +basejump/bsg_idiv_iterative_controller.sv +basejump/bsg_idiv_iterative.sv +pyc_word_operand_normalize.sv +pyc_word_result_normalize.sv +pyc_div_special_cases.sv +pyc_runtime_div.sv +pyc_runtime_div_packet.sv diff --git a/library/verilog/div_pto_v2/div_pto_v2/pyc_div_special_cases.sv b/library/verilog/div_pto_v2/div_pto_v2/pyc_div_special_cases.sv new file mode 100644 index 000000000..7578ddbaa --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/pyc_div_special_cases.sv @@ -0,0 +1,80 @@ +// PTO scalar divide/remainder special-case detector. +// +// The PTO scalar ALU semantics are total: +// divisor == 0: +// quotient = 0 +// remainder = dividend +// signed minimum / -1: +// quotient = signed minimum +// remainder = 0 +// +// Inputs lhs_normalized/rhs_normalized must already reflect W-form operand +// normalization when word_mode=1. Outputs are raw WIDTH-bit values; W-form +// result sign-extension is applied by pyc_word_result_normalize afterwards. +module pyc_div_special_cases #( + parameter integer WIDTH = 64, + parameter integer WORD_WIDTH = 32 +) ( + input wire [WIDTH-1:0] lhs, + input wire [WIDTH-1:0] rhs, + input wire [WIDTH-1:0] lhs_normalized, + input wire [WIDTH-1:0] rhs_normalized, + input wire is_signed, + input wire word_mode, + + output wire is_special, + output wire divide_by_zero, + output wire signed_overflow, + output wire [WIDTH-1:0] quotient_raw, + output wire [WIDTH-1:0] remainder_raw +); + + localparam [WIDTH-1:0] FULL_MIN = {1'b1, {(WIDTH-1){1'b0}}}; + localparam [WIDTH-1:0] FULL_NEG_ONE = {WIDTH{1'b1}}; + + wire full_signed_overflow; + wire word_signed_overflow; + + assign divide_by_zero = (rhs_normalized == {WIDTH{1'b0}}); + + assign full_signed_overflow = is_signed && !word_mode && + (lhs == FULL_MIN) && + (rhs == FULL_NEG_ONE); + + generate + if (WORD_WIDTH == 32) begin : gen_word32_overflow + assign word_signed_overflow = is_signed && word_mode && + (lhs[31:0] == 32'h8000_0000) && + (rhs[31:0] == 32'hffff_ffff); + end else begin : gen_generic_word_overflow + wire [WORD_WIDTH-1:0] word_min; + wire [WORD_WIDTH-1:0] word_neg_one; + assign word_min = {1'b1, {(WORD_WIDTH-1){1'b0}}}; + assign word_neg_one = {WORD_WIDTH{1'b1}}; + assign word_signed_overflow = is_signed && word_mode && + (lhs[WORD_WIDTH-1:0] == word_min) && + (rhs[WORD_WIDTH-1:0] == word_neg_one); + end + endgenerate + + assign signed_overflow = full_signed_overflow || word_signed_overflow; + assign is_special = divide_by_zero || signed_overflow; + + // Zero-divisor semantics take priority. The two special cases cannot + // overlap in legal arithmetic, but the priority keeps the logic explicit. + assign quotient_raw = divide_by_zero ? {WIDTH{1'b0}} + : signed_overflow ? lhs_normalized + : {WIDTH{1'b0}}; + + assign remainder_raw = divide_by_zero ? lhs_normalized + : signed_overflow ? {WIDTH{1'b0}} + : {WIDTH{1'b0}}; + +`ifndef SYNTHESIS + initial begin + if (WIDTH < WORD_WIDTH) + $error("pyc_div_special_cases requires WIDTH >= WORD_WIDTH"); + end +`endif + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div.sv b/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div.sv new file mode 100644 index 000000000..57b95e3d4 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div.sv @@ -0,0 +1,236 @@ +// Canonical PTO scalar DIV/REM runtime primitive built around BaseJump's +// iterative divider. +// +// Supported normalized operation families: +// DIV / REM : is_signed=1, word_mode=0 +// DIVU / REMU : is_signed=0, word_mode=0 +// DIVW / REMW : is_signed=1, word_mode=1 +// DIVUW / REMUW : is_signed=0, word_mode=1 +// +// One physical divider produces quotient and remainder together. The caller +// selects which result belongs to the architectural mnemonic. Divide-by-zero +// and signed-minimum/-1 are handled at the wrapper boundary so the PTO total +// semantics do not depend on the imported divider's corner-case behavior. +module pyc_runtime_div #( + parameter integer WIDTH = 64, + parameter integer WORD_WIDTH = 32, + parameter integer BITS_PER_ITER = 1 +) ( + input wire clk, + input wire reset, + + input wire request_valid, + output wire request_ready, + input wire [WIDTH-1:0] dividend, + input wire [WIDTH-1:0] divisor, + input wire is_signed, + input wire word_mode, + + output wire response_valid, + input wire response_ready, + output wire [WIDTH-1:0] quotient, + output wire [WIDTH-1:0] remainder, + + // Informational only. PTO divide-by-zero is a defined arithmetic result, + // not an architectural fault. This port is retained for compatibility and + // diagnostics; normal execution should consume quotient/remainder. + output wire divide_by_zero +); + + // -------------------------------------------------------------------------- + // Fixed-word operand normalization (not wrapping-bitfield normalization) + // -------------------------------------------------------------------------- + wire [WIDTH-1:0] dividend_normalized; + wire [WIDTH-1:0] divisor_normalized; + + pyc_word_operand_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_dividend ( + .value(dividend), + .word_mode(word_mode), + .signed_word(is_signed), + .normalized(dividend_normalized) + ); + + pyc_word_operand_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_divisor ( + .value(divisor), + .word_mode(word_mode), + .signed_word(is_signed), + .normalized(divisor_normalized) + ); + + // -------------------------------------------------------------------------- + // PTO-defined arithmetic special cases + // -------------------------------------------------------------------------- + wire special_request; + wire request_divide_by_zero; + wire request_signed_overflow; + wire [WIDTH-1:0] special_quotient_raw; + wire [WIDTH-1:0] special_remainder_raw; + + pyc_div_special_cases #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) special_cases ( + .lhs(dividend), + .rhs(divisor), + .lhs_normalized(dividend_normalized), + .rhs_normalized(divisor_normalized), + .is_signed(is_signed), + .word_mode(word_mode), + .is_special(special_request), + .divide_by_zero(request_divide_by_zero), + .signed_overflow(request_signed_overflow), + .quotient_raw(special_quotient_raw), + .remainder_raw(special_remainder_raw) + ); + + wire [WIDTH-1:0] special_quotient_final; + wire [WIDTH-1:0] special_remainder_final; + + pyc_word_result_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_special_quotient ( + .value(special_quotient_raw), + .word_mode(word_mode), + .normalized(special_quotient_final) + ); + + pyc_word_result_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_special_remainder ( + .value(special_remainder_raw), + .word_mode(word_mode), + .normalized(special_remainder_final) + ); + + // -------------------------------------------------------------------------- + // Special-result holding register. + // A special request bypasses the iterative divider and produces a stable + // response that remains valid until response_ready is asserted. + // -------------------------------------------------------------------------- + reg special_valid_r; + reg [WIDTH-1:0] special_quotient_r; + reg [WIDTH-1:0] special_remainder_r; + reg special_divide_by_zero_r; + + // -------------------------------------------------------------------------- + // BaseJump core + // -------------------------------------------------------------------------- + wire core_ready; + wire core_valid; + wire [WIDTH-1:0] core_quotient_raw; + wire [WIDTH-1:0] core_remainder_raw; + + wire request_fire; + wire launch_core; + wire consume_core; + + // There is at most one outstanding result in this wrapper. The BaseJump + // core reports ready only when idle; a held special result independently + // blocks new requests. + assign request_ready = core_ready && !special_valid_r; + assign request_fire = request_valid && request_ready; + assign launch_core = request_fire && !special_request; + + // Capture word_mode for the normal divider request because the core itself + // does not return request metadata with its eventual result. + reg core_word_mode_r; + + always @(posedge clk) begin + if (reset) begin + special_valid_r <= 1'b0; + special_quotient_r <= {WIDTH{1'b0}}; + special_remainder_r <= {WIDTH{1'b0}}; + special_divide_by_zero_r <= 1'b0; + core_word_mode_r <= 1'b0; + end else begin + if (special_valid_r && response_ready) begin + special_valid_r <= 1'b0; + special_divide_by_zero_r <= 1'b0; + end + + if (request_fire && special_request) begin + special_valid_r <= 1'b1; + special_quotient_r <= special_quotient_final; + special_remainder_r <= special_remainder_final; + special_divide_by_zero_r <= request_divide_by_zero; + end + + if (launch_core) + core_word_mode_r <= word_mode; + end + end + + assign consume_core = core_valid && response_ready && !special_valid_r; + + bsg_idiv_iterative #( + .width_p(WIDTH), + .bitstack_p(0), + .bits_per_iter_p(BITS_PER_ITER) + ) impl ( + .clk_i(clk), + .reset_i(reset), + .v_i(launch_core), + .ready_and_o(core_ready), + .dividend_i(dividend_normalized), + .divisor_i(divisor_normalized), + .signed_div_i(is_signed), + .v_o(core_valid), + .quotient_o(core_quotient_raw), + .remainder_o(core_remainder_raw), + .yumi_i(consume_core) + ); + + // -------------------------------------------------------------------------- + // PTO W-form result normalization. Unsigned W forms also sign-extend the + // low 32 result bits to XLEN, exactly as the ASL semantics specify. + // -------------------------------------------------------------------------- + wire [WIDTH-1:0] core_quotient_final; + wire [WIDTH-1:0] core_remainder_final; + + pyc_word_result_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_core_quotient ( + .value(core_quotient_raw), + .word_mode(core_word_mode_r), + .normalized(core_quotient_final) + ); + + pyc_word_result_normalize #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH) + ) normalize_core_remainder ( + .value(core_remainder_raw), + .word_mode(core_word_mode_r), + .normalized(core_remainder_final) + ); + + assign response_valid = special_valid_r || core_valid; + assign quotient = special_valid_r ? special_quotient_r + : core_quotient_final; + assign remainder = special_valid_r ? special_remainder_r + : core_remainder_final; + assign divide_by_zero = special_valid_r && special_divide_by_zero_r; + + // request_signed_overflow is intentionally not exported: PTO treats it as a + // fully defined arithmetic result rather than an exception/fault condition. + wire _unused_request_signed_overflow = request_signed_overflow; + +`ifndef SYNTHESIS + initial begin + if (WIDTH < WORD_WIDTH) + $error("pyc_runtime_div requires WIDTH >= WORD_WIDTH"); + if (!(BITS_PER_ITER == 1 || BITS_PER_ITER == 2)) + $error("pyc_runtime_div BITS_PER_ITER must be 1 or 2"); + end +`endif + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div_packet.sv b/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div_packet.sv new file mode 100644 index 000000000..b2f91f5e0 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/pyc_runtime_div_packet.sv @@ -0,0 +1,84 @@ +// Ready/valid packet adapter for the shared PTO iterative divider. +// +// PYC queue graphs expose semantic values as pure SSA expressions, while the +// qualified divider is a variable-latency ready/valid unit. This adapter +// bridges those contracts by retaining the complete input packet until the +// quotient/remainder response is accepted. The packet itself is deliberately +// opaque here; the generated PYC wiring supplies the operand and mode slices. +module pyc_runtime_div_packet #( + parameter integer WIDTH = 64, + parameter integer WORD_WIDTH = 32, + parameter integer PACKET_WIDTH = 1, + parameter integer BITS_PER_ITER = 1 +) ( + input wire clk, + input wire reset, + + input wire request_valid, + output wire request_ready, + input wire [PACKET_WIDTH-1:0] packet_in, + input wire [WIDTH-1:0] dividend, + input wire [WIDTH-1:0] divisor, + input wire is_signed, + input wire word_mode, + + output wire response_valid, + input wire response_ready, + output wire [PACKET_WIDTH-1:0] packet_out, + output wire [WIDTH-1:0] quotient, + output wire [WIDTH-1:0] remainder +); + + wire divider_request_ready; + wire divider_response_valid; + wire divider_response_ready; + wire divide_by_zero; + wire request_fire; + + reg packet_valid_r; + reg [PACKET_WIDTH-1:0] packet_r; + + // Only one packet may be outstanding. The input FIFO therefore holds its + // output stable until this adapter accepts the request. + assign request_ready = divider_request_ready && !packet_valid_r; + assign request_fire = request_valid && request_ready; + + assign response_valid = packet_valid_r && divider_response_valid; + assign divider_response_ready = response_ready && packet_valid_r; + assign packet_out = packet_valid_r ? packet_r : packet_in; + + always @(posedge clk) begin + if (reset) begin + packet_valid_r <= 1'b0; + packet_r <= {PACKET_WIDTH{1'b0}}; + end else begin + if (response_valid && response_ready) + packet_valid_r <= 1'b0; + if (request_fire) begin + packet_valid_r <= 1'b1; + packet_r <= packet_in; + end + end + end + + pyc_runtime_div #( + .WIDTH(WIDTH), + .WORD_WIDTH(WORD_WIDTH), + .BITS_PER_ITER(BITS_PER_ITER) + ) divider ( + .clk(clk), + .reset(reset), + .request_valid(request_fire), + .request_ready(divider_request_ready), + .dividend(dividend), + .divisor(divisor), + .is_signed(is_signed), + .word_mode(word_mode), + .response_valid(divider_response_valid), + .response_ready(divider_response_ready), + .quotient(quotient), + .remainder(remainder), + .divide_by_zero(divide_by_zero) + ); + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/pyc_word_operand_normalize.sv b/library/verilog/div_pto_v2/div_pto_v2/pyc_word_operand_normalize.sv new file mode 100644 index 000000000..06acf4934 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/pyc_word_operand_normalize.sv @@ -0,0 +1,43 @@ +// Canonical fixed-word operand normalizer for PTO scalar W-form operations. +// +// word_mode=0: pass the full WIDTH-bit value through unchanged. +// word_mode=1, signed_word=0: zero-extend the low WORD_WIDTH bits. +// word_mode=1, signed_word=1: sign-extend the low WORD_WIDTH bits. +// +// This helper is intentionally not a wrapping-bitfield operator: W-form +// arithmetic always uses the fixed low word, with no runtime offset or wrap. +module pyc_word_operand_normalize #( + parameter integer WIDTH = 64, + parameter integer WORD_WIDTH = 32 +) ( + input wire [WIDTH-1:0] value, + input wire word_mode, + input wire signed_word, + output wire [WIDTH-1:0] normalized +); + + generate + if (WIDTH > WORD_WIDTH) begin : gen_extend + wire [WIDTH-1:0] zero_extended; + wire [WIDTH-1:0] sign_extended; + + assign zero_extended = {{(WIDTH-WORD_WIDTH){1'b0}}, + value[WORD_WIDTH-1:0]}; + assign sign_extended = {{(WIDTH-WORD_WIDTH){value[WORD_WIDTH-1]}}, + value[WORD_WIDTH-1:0]}; + assign normalized = word_mode + ? (signed_word ? sign_extended : zero_extended) + : value; + end else begin : gen_same_width + assign normalized = value; + end + endgenerate + +`ifndef SYNTHESIS + initial begin + if (WIDTH < WORD_WIDTH) + $error("pyc_word_operand_normalize requires WIDTH >= WORD_WIDTH"); + end +`endif + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/pyc_word_result_normalize.sv b/library/verilog/div_pto_v2/div_pto_v2/pyc_word_result_normalize.sv new file mode 100644 index 000000000..46c6fd07f --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/pyc_word_result_normalize.sv @@ -0,0 +1,32 @@ +// Canonical PTO scalar W-form result normalizer. +// +// PTO DIVW/DIVUW/REMW/REMUW all sign-extend the low 32-bit result to XLEN, +// including the unsigned W forms. word_mode=0 passes the full result through. +module pyc_word_result_normalize #( + parameter integer WIDTH = 64, + parameter integer WORD_WIDTH = 32 +) ( + input wire [WIDTH-1:0] value, + input wire word_mode, + output wire [WIDTH-1:0] normalized +); + + generate + if (WIDTH > WORD_WIDTH) begin : gen_extend + wire [WIDTH-1:0] sign_extended; + assign sign_extended = {{(WIDTH-WORD_WIDTH){value[WORD_WIDTH-1]}}, + value[WORD_WIDTH-1:0]}; + assign normalized = word_mode ? sign_extended : value; + end else begin : gen_same_width + assign normalized = value; + end + endgenerate + +`ifndef SYNTHESIS + initial begin + if (WIDTH < WORD_WIDTH) + $error("pyc_word_result_normalize requires WIDTH >= WORD_WIDTH"); + end +`endif + +endmodule diff --git a/library/verilog/div_pto_v2/div_pto_v2/tests/files_tb.f b/library/verilog/div_pto_v2/div_pto_v2/tests/files_tb.f new file mode 100644 index 000000000..5fcafa22f --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/tests/files_tb.f @@ -0,0 +1,2 @@ +-f files.f +tests/tb_pyc_runtime_div.sv diff --git a/library/verilog/div_pto_v2/div_pto_v2/tests/tb_pyc_runtime_div.sv b/library/verilog/div_pto_v2/div_pto_v2/tests/tb_pyc_runtime_div.sv new file mode 100644 index 000000000..de793ee96 --- /dev/null +++ b/library/verilog/div_pto_v2/div_pto_v2/tests/tb_pyc_runtime_div.sv @@ -0,0 +1,140 @@ +`timescale 1ns/1ps + +module tb_pyc_runtime_div; + localparam integer WIDTH = 64; + + reg clk; + reg reset; + reg request_valid; + wire request_ready; + reg [WIDTH-1:0] dividend; + reg [WIDTH-1:0] divisor; + reg is_signed; + reg word_mode; + wire response_valid; + reg response_ready; + wire [WIDTH-1:0] quotient; + wire [WIDTH-1:0] remainder; + wire divide_by_zero; + + pyc_runtime_div #( + .WIDTH(64), + .WORD_WIDTH(32), + .BITS_PER_ITER(1) + ) dut ( + .clk(clk), + .reset(reset), + .request_valid(request_valid), + .request_ready(request_ready), + .dividend(dividend), + .divisor(divisor), + .is_signed(is_signed), + .word_mode(word_mode), + .response_valid(response_valid), + .response_ready(response_ready), + .quotient(quotient), + .remainder(remainder), + .divide_by_zero(divide_by_zero) + ); + + always #5 clk = ~clk; + + task automatic run_case; + input [255:0] name; + input [63:0] lhs; + input [63:0] rhs; + input signed_mode; + input word; + input [63:0] expected_q; + input [63:0] expected_r; + input expected_divzero; + begin + while (!request_ready) @(posedge clk); + dividend <= lhs; + divisor <= rhs; + is_signed <= signed_mode; + word_mode <= word; + request_valid <= 1'b1; + @(posedge clk); + request_valid <= 1'b0; + + while (!response_valid) @(posedge clk); + if (quotient !== expected_q || + remainder !== expected_r || + divide_by_zero !== expected_divzero) begin + $display("FAIL %s", name); + $display(" got q=%h r=%h div0=%b", quotient, remainder, + divide_by_zero); + $display(" exp q=%h r=%h div0=%b", expected_q, expected_r, + expected_divzero); + $fatal(1); + end + $display("PASS %s q=%h r=%h", name, quotient, remainder); + @(posedge clk); + end + endtask + + initial begin + clk = 1'b0; + reset = 1'b1; + request_valid = 1'b0; + dividend = 64'b0; + divisor = 64'b0; + is_signed = 1'b0; + word_mode = 1'b0; + response_ready = 1'b1; + + repeat (3) @(posedge clk); + reset <= 1'b0; + @(posedge clk); + + // XLEN signed/unsigned basics. + run_case("DIV 10/3", 64'd10, 64'd3, 1'b1, 1'b0, + 64'd3, 64'd1, 1'b0); + run_case("DIVU 15/4", 64'd15, 64'd4, 1'b0, 1'b0, + 64'd3, 64'd3, 1'b0); + + // PTO total divide-by-zero semantics: quotient=0, remainder=dividend. + run_case("DIV by zero", 64'hffff_ffff_ffff_fff6, 64'd0, + 1'b1, 1'b0, + 64'd0, 64'hffff_ffff_ffff_fff6, 1'b1); + + // Signed minimum / -1: minimum quotient, zero remainder. + run_case("DIV signed overflow", 64'h8000_0000_0000_0000, + 64'hffff_ffff_ffff_ffff, + 1'b1, 1'b0, + 64'h8000_0000_0000_0000, 64'd0, 1'b0); + + // Word signed division: -10 / 3 = -3 remainder -1, both sign-extended. + run_case("DIVW -10/3", 64'h0000_0000_ffff_fff6, 64'd3, + 1'b1, 1'b1, + 64'hffff_ffff_ffff_fffd, + 64'hffff_ffff_ffff_ffff, + 1'b0); + + // Unsigned W result is still sign-extended from low 32 bits by PTO. + run_case("DIVUW ffffffff/1", 64'hffff_ffff_ffff_ffff, 64'd1, + 1'b0, 1'b1, + 64'hffff_ffff_ffff_ffff, + 64'd0, + 1'b0); + + // Zero divisor in unsigned W form: low-32 dividend then sign-extend result. + run_case("REMUW zero divisor", 64'h1234_5678_ffff_ffff, 64'd0, + 1'b0, 1'b1, + 64'd0, + 64'hffff_ffff_ffff_ffff, + 1'b1); + + // Signed W minimum / -1. + run_case("DIVW signed overflow", 64'h0000_0000_8000_0000, + 64'h0000_0000_ffff_ffff, + 1'b1, 1'b1, + 64'hffff_ffff_8000_0000, + 64'd0, + 1'b0); + + $display("ALL PTO DIV/REM DIRECTED CASES PASSED"); + $finish; + end +endmodule diff --git a/library/verilog/rtl_catalog.json b/library/verilog/rtl_catalog.json index 3670f665c..1176ca8ed 100644 --- a/library/verilog/rtl_catalog.json +++ b/library/verilog/rtl_catalog.json @@ -20,8 +20,13 @@ "WIDTH": "input_width" }, "ports": { - "inputs": ["in_value"], - "outputs": ["index", "valid"] + "inputs": [ + "in_value" + ], + "outputs": [ + "index", + "valid" + ] }, "sources": [ { @@ -56,15 +61,19 @@ "WIDTH": "input_width" }, "ports": { - "inputs": ["in_value"], - "outputs": ["count"] + "inputs": [ + "in_value" + ], + "outputs": [ + "count" + ] }, "sources": [ { - "path": "pyc_popcount_primitive.v", - "sha256": "sha256:e5ab6d5c1f5657d4b43f121ee05738f42160c66c7d997059fd2893fdb75ebce4", + "path": "bitfield_primitives/pyc_popcount_primitive.sv", + "sha256": "sha256:019567028940095d7aa39d095078112ed15a45829dc3de792919426f8e46ef02", "license": "BSD-3-Clause", - "modified": false + "modified": true } ], "license_file": "licenses/BSD-3-Clause.txt", @@ -76,7 +85,7 @@ }, "qualification": { "status": "validated", - "report": "docs/gates/logs/20260905-pyc-popcount-rtl-r1/summary.md" + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" } }, { @@ -99,8 +108,12 @@ "WIDTH": "input_width" }, "ports": { - "inputs": ["in_value"], - "outputs": ["count"] + "inputs": [ + "in_value" + ], + "outputs": [ + "count" + ] }, "sources": [ { @@ -121,6 +134,296 @@ "status": "validated", "report": "docs/gates/logs/20260905-pyc-zero-count-family-r1/summary.md" } + }, + { + "semantic_id": "pyc.wrapping_field_normalize.v1", + "implementation_id": "pyc.bsd.wrapping_field_normalize.v1", + "effect_class": "comb", + "module": "pyc_wrapping_field_normalize", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "CONTROL_WIDTH": "width_control", + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "value", + "bit_width", + "bit_offset" + ], + "outputs": [ + "field", + "mask" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_wrapping_field_normalize.sv", + "sha256": "sha256:05216c60237a96a3bbd2f0b2920a10808fb84d4d6a9574f906692d034d8867d8", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.bitfield_clear.v1", + "implementation_id": "pyc.bsd.bitfield_clear.v1", + "effect_class": "comb", + "module": "pyc_bitfield_clear", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "value", + "mask" + ], + "outputs": [ + "result" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_bitfield_clear.sv", + "sha256": "sha256:faa64d52b0250eda3f2e6b7706f50ecda7607700f25867a89b06ebc1a8551d8e", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.bitfield_set.v1", + "implementation_id": "pyc.bsd.bitfield_set.v1", + "effect_class": "comb", + "module": "pyc_bitfield_set", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "value", + "mask" + ], + "outputs": [ + "result" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_bitfield_set.sv", + "sha256": "sha256:4f9799028e14ec88692c1fe95239c537a81cadab39093edd5e7cb7220fdd2326", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.bitfield_insert.v1", + "implementation_id": "pyc.bsd.bitfield_insert.v1", + "effect_class": "comb", + "module": "pyc_bitfield_insert", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "CONTROL_WIDTH": "width_control", + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "value", + "source", + "mask", + "bit_width", + "bit_offset" + ], + "outputs": [ + "result" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_bitfield_insert.sv", + "sha256": "sha256:44bdce1cd509eb20863d192ffc7794098644f340c235663fc211ae7c9ff294d6", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.reverse_bytes.v1", + "implementation_id": "pyc.bsd.reverse_bytes.v1", + "effect_class": "comb", + "module": "pyc_reverse_bytes", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "CONTROL_WIDTH": "width_control", + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "field", + "bit_width" + ], + "outputs": [ + "result" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_reverse_bytes.sv", + "sha256": "sha256:04ac0fe466de318b1b4c6af4c9a5fb8e202b4679497999acfe7e7afed1f6eb37", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.dynamic_sign_extend.v1", + "implementation_id": "pyc.bsd.dynamic_sign_extend.v1", + "effect_class": "comb", + "module": "pyc_dynamic_sign_extend", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "CONTROL_WIDTH": "width_control", + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "field", + "bit_width" + ], + "outputs": [ + "result" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_dynamic_sign_extend.sv", + "sha256": "sha256:64a96ed7859f7c453cd1fb30600ce674d985ace0c46ee0669f4b1156a3571fc1", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } + }, + { + "semantic_id": "pyc.runtime_zero_count.v1", + "implementation_id": "pyc.bsd.runtime_zero_count.v1", + "effect_class": "comb", + "module": "pyc_runtime_zero_count", + "min_width": 1, + "max_width": 64, + "selection_priority": 100, + "parameter_bindings": { + "COUNT_WIDTH": "output_width", + "WIDTH": "input_width" + }, + "ports": { + "inputs": [ + "value", + "direction_low" + ], + "outputs": [ + "count" + ] + }, + "sources": [ + { + "path": "bitfield_primitives/pyc_runtime_zero_count.sv", + "sha256": "sha256:666bbc8290fe87b505eaa1604e1579186c5c9e830083069a25041e40d0c5eec8", + "license": "BSD-3-Clause", + "modified": true + } + ], + "license_file": "licenses/BSD-3-Clause.txt", + "license_sha256": "sha256:321940059773d631ad73832bfa870431aa6cba545b38d4055641ce21275bb22c", + "provenance": { + "origin": "PTO-ISA/pyCircuit", + "design_input": "ALU wrapping-bitfield decomposition", + "license": "BSD-3-Clause" + }, + "qualification": { + "status": "validated", + "report": "docs/gates/logs/20260916-alu-bitfield-decomposition/qualification.json" + } } ] } diff --git a/python/agentic-circuit/src/agentic_circuit/__init__.py b/python/agentic-circuit/src/agentic_circuit/__init__.py index fe8ddeab0..30246f82e 100644 --- a/python/agentic-circuit/src/agentic_circuit/__init__.py +++ b/python/agentic-circuit/src/agentic_circuit/__init__.py @@ -46,7 +46,18 @@ s64, ) from .markers import ( + addw, + andw, barrier, + bitfield_clear, + bitfield_clz, + bitfield_ctz, + bitfield_extract, + bitfield_insert, + bitfield_popcount, + bitfield_reverse_bytes, + bitfield_set, + csel, compute, concat, count_leading_zeros, @@ -54,13 +65,18 @@ engine, expect, fork, + divrem, insert, instances, map, + madd, + maddw, match_enum, matches, memory, merge, + msub, + mulw, observe, onehot_encode, onehot_enum, @@ -72,14 +88,30 @@ route, schedule, scope, + sdiv, sext, + sext_low, set, sink, slot, source, + sll, + sllw, + smax, + smin, + sra, + sraw, + srem, + srl, + srlw, + subw, table, static_assert, truncate, + udiv, + umax, + umin, + urem, wrap, saturate, checked, @@ -87,6 +119,9 @@ view, zero, zext, + zext_low, + orw, + xorw, ) _UNSIGNED_NAMES = tuple(f"u{width}" for width in _types_module.UNSIGNED_WIDTHS) diff --git a/python/agentic-circuit/src/agentic_circuit/_queue_frontend.py b/python/agentic-circuit/src/agentic_circuit/_queue_frontend.py index 6a914f8ca..e44f04920 100644 --- a/python/agentic-circuit/src/agentic_circuit/_queue_frontend.py +++ b/python/agentic-circuit/src/agentic_circuit/_queue_frontend.py @@ -289,18 +289,14 @@ def _constant_integer( result = ( left + right if isinstance(node.op, ast.Add) - else left - right - if isinstance(node.op, ast.Sub) - else left * right + else left - right if isinstance(node.op, ast.Sub) else left * right ) return result if -(1 << 63) <= result <= (1 << 63) - 1 else None if isinstance(node, ast.Call) and len(node.args) == 1 and not node.keywords: helper = ( node.func.attr if isinstance(node.func, ast.Attribute) - else node.func.id - if isinstance(node.func, ast.Name) - else "" + else node.func.id if isinstance(node.func, ast.Name) else "" ) if helper in {"index_width", "count_width"}: operand = _constant_integer(node.args[0], values) @@ -642,9 +638,7 @@ def _static_parameter_aliases(tree: ast.Module) -> dict[str, StaticParameterAlia family_name = ( family.value.attr if isinstance(family.value, ast.Attribute) - else family.value.id - if isinstance(family.value, ast.Name) - else "" + else family.value.id if isinstance(family.value, ast.Name) else "" ) parameter_type = family.slice.id if isinstance(family.slice, ast.Name) else "" if family_name != "param": @@ -2876,9 +2870,9 @@ def _pure_helper_definitions( reachable_helpers.add(item.id) pending_helpers.append(item.id) nodes: dict[str, ast.FunctionDef] = {} - signatures: dict[ - str, tuple[tuple[tuple[str, ValueType], ...], ValueType, bool] - ] = {} + signatures: dict[str, tuple[tuple[tuple[str, ValueType], ...], ValueType, bool]] = ( + {} + ) for node in tree.body: if not isinstance(node, ast.FunctionDef): continue @@ -4227,9 +4221,7 @@ def parse_optional_multi_output_rule( spelling = ( candidate.func.attr if isinstance(candidate.func, ast.Attribute) - else candidate.func.id - if isinstance(candidate.func, ast.Name) - else None + else candidate.func.id if isinstance(candidate.func, ast.Name) else None ) if spelling in forbidden_runtime_mechanics: raise QueueFrontendError( @@ -5962,9 +5954,7 @@ def system_result_payloads( returned_values = ( tuple(returned.elts) if isinstance(returned, (ast.Tuple, ast.List)) - else (returned,) - if returned is not None - else () + else (returned,) if returned is not None else () ) if len(returned_values) == len(result_payloads) and all( isinstance(value, ast.Name) for value in returned_values @@ -11384,6 +11374,7 @@ def collect_enums(descriptor: ValueType) -> None: self.lines: list[str] = [] self.index = 0 self.priority_values: dict[str, tuple[str, ValueType, str, ValueType]] = {} + self.divrem_values: dict[str, tuple[tuple[str, str], TupleType]] = {} self.onehot_values: dict[ str, tuple[ @@ -13677,6 +13668,25 @@ def _emit_node( ) return name, aggregate if isinstance(node, ast.Subscript): + if ( + isinstance(node.value, ast.Call) + and _decorator_name(node.value.func).rsplit(".", 1)[-1] == "divrem" + ): + index = _constant_integer(node.slice) + if index not in (0, 1): + raise QueueFrontendError( + "ACPY-DIV-002: divrem result index must be 0 or 1" + ) + key = ast.dump(node.value, include_attributes=False) + pair = self.divrem_values.get(key) + if pair is None: + self.emit(node.value) + pair = self.divrem_values.get(key) + if pair is None: + raise QueueFrontendError( + "ACPY-DIV-002: divrem result is unavailable" + ) + return self._remember(pair[0][index], BitsType(64)) view = self._bitfield_view(node.value) if view is not None: schema_name, layout, base_node = view @@ -14232,9 +14242,7 @@ def _emit_node( else ( expected if not self.strict_descriptors and expected is not None - else BoolType() - if type(node.value) is bool - else BitsType(64) + else BoolType() if type(node.value) is bool else BitsType(64) ) ) ) @@ -14248,9 +14256,7 @@ def _emit_node( value = ( "true" if node.value is True - else "false" - if node.value is False - else str(node.value) + else "false" if node.value is False else str(node.value) ) attribute_type = ( f"i{typ.width}" if isinstance(typ, RangeType) else _render_type(typ) @@ -14733,6 +14739,309 @@ def _emit_node( f"!ac.var<{_render_type(left_type)}> -> !ac.var" ) return name, BoolType() + if isinstance(node, ast.Call): + operation = _decorator_name(node.func).rsplit(".", 1)[-1] + if operation in {"udiv", "sdiv", "urem", "srem"}: + if len(node.args) != 2 or node.keywords: + raise QueueFrontendError( + f"ACPY-QUEUE-003: {operation} requires two positional operands" + ) + left, left_type = self.emit(node.args[0]) + right, right_type = self.emit(node.args[1], left_type) + if not self._types_match(left_type, right_type): + raise QueueFrontendError( + f"ACPY-QUEUE-003: {operation} operands must match" + ) + if _epoch_05_integer_width(left_type) is None: + raise QueueFrontendError( + f"ACPY-QUEUE-003: {operation} operands must be integer payloads" + ) + name = self._new() + self.lines.append( + f" %{name} = ac.var.{operation} %{left}, %{right} : " + f"!ac.var<{_render_type(left_type)}>" + ) + return self._remember(name, left_type, Unknown()) + if operation == "divrem": + if len(node.args) != 2 or len(node.keywords) != 2: + raise QueueFrontendError( + "ACPY-DIV-001: divrem requires lhs, rhs, signed, and word" + ) + keyword_values = { + keyword.arg: keyword.value + for keyword in node.keywords + if keyword.arg is not None + } + if set(keyword_values) != {"signed", "word"}: + raise QueueFrontendError( + "ACPY-DIV-001: divrem requires signed and word keywords" + ) + left, left_type = self.emit(node.args[0]) + right, right_type = self.emit(node.args[1], left_type) + if not self._types_match(left_type, right_type): + raise QueueFrontendError("ACPY-DIV-001: divrem operands must match") + if _epoch_05_integer_width(left_type) != 64: + raise QueueFrontendError( + "ACPY-DIV-001: divrem operands must be i64 integers" + ) + signed, signed_type = self.emit(keyword_values["signed"], BoolType()) + word, word_type = self.emit(keyword_values["word"], BoolType()) + if not self._types_match( + signed_type, BoolType() + ) or not self._types_match(word_type, BoolType()): + raise QueueFrontendError( + "ACPY-DIV-001: divrem signed/word must be bool values" + ) + key = ast.dump(node, include_attributes=False) + cached = self.divrem_values.get(key) + if cached is not None: + return f"__divrem_{key}", cached[1] + quotient = self._new() + remainder = self._new() + self.lines.append( + f" %{quotient}, %{remainder} = ac.var.divrem %{left}, %{right}, " + f"%{signed}, %{word} : " + f"!ac.var<{_render_type(left_type)}>, " + f"!ac.var<{_render_type(right_type)}>, " + f"!ac.var<{_render_type(signed_type)}>, " + f"!ac.var<{_render_type(word_type)}> -> " + "!ac.var, !ac.var" + ) + pair_type = TupleType((BitsType(64), BitsType(64))) + self.divrem_values[key] = ((quotient, remainder), pair_type) + return f"__divrem_{key}", pair_type + + binary_operations = { + "addw", + "subw", + "andw", + "orw", + "xorw", + "sll", + "srl", + "sra", + "sllw", + "srlw", + "sraw", + "smin", + "umin", + "smax", + "umax", + "mulw", + } + if operation in binary_operations: + if len(node.args) != 2 or node.keywords: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} requires two positional operands" + ) + left, left_type = self.emit(node.args[0]) + right, right_type = self.emit(node.args[1], left_type) + if not self._types_match(left_type, right_type): + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must match" + ) + if _epoch_05_integer_width(left_type) is None: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must be integer payloads" + ) + name = self._new() + rendered = _render_type(left_type) + self.lines.append( + f" %{name} = ac.var.{operation} %{left}, %{right} : " + f"!ac.var<{rendered}> -> !ac.var<{rendered}>" + ) + return self._remember(name, left_type, Unknown()) + + if operation in {"madd", "maddw", "msub"}: + if len(node.args) != 3 or node.keywords: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} requires three positional operands" + ) + left, left_type = self.emit(node.args[0]) + right, right_type = self.emit(node.args[1], left_type) + auxiliary, auxiliary_type = self.emit(node.args[2], left_type) + if not ( + self._types_match(left_type, right_type) + and self._types_match(left_type, auxiliary_type) + ): + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must match" + ) + if _epoch_05_integer_width(left_type) is None: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must be integer payloads" + ) + name = self._new() + rendered = _render_type(left_type) + self.lines.append( + f" %{name} = ac.var.{operation} %{left}, %{right}, " + f"%{auxiliary} : !ac.var<{rendered}> -> !ac.var<{rendered}>" + ) + return self._remember(name, left_type, Unknown()) + + if operation == "bitfield_extract": + signed_keywords = [ + keyword.value + for keyword in node.keywords + if keyword.arg == "signed" + ] + if ( + len(node.args) != 3 + or len(node.keywords) != 1 + or len(signed_keywords) != 1 + or not isinstance(signed_keywords[0], ast.Constant) + or type(signed_keywords[0].value) is not bool + ): + raise QueueFrontendError( + "ACPY-ALU-001: bitfield_extract signed must be static bool" + ) + value, value_type = self.emit(node.args[0]) + width, width_type = self.emit(node.args[1]) + offset, offset_type = self.emit(node.args[2]) + if _epoch_05_integer_width(value_type) is None or any( + _epoch_05_integer_width(descriptor) is None + for descriptor in (width_type, offset_type) + ): + raise QueueFrontendError( + "ACPY-ALU-001: bitfield_extract operands must be integers" + ) + name = self._new() + rendered = _render_type(value_type) + self.lines.append( + f" %{name} = ac.var.bitfield_extract %{value}, %{width}, " + f"%{offset} signed_mode " + f"{'true' if signed_keywords[0].value else 'false'} : " + f"!ac.var<{rendered}>, !ac.var<{_render_type(width_type)}>, " + f"!ac.var<{_render_type(offset_type)}> -> !ac.var<{rendered}>" + ) + return self._remember(name, value_type, Unknown()) + + bitfield_operations = { + "bitfield_popcount", + "bitfield_clz", + "bitfield_ctz", + "bitfield_clear", + "bitfield_set", + "bitfield_reverse_bytes", + } + if operation in bitfield_operations: + if len(node.args) != 3 or node.keywords: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} requires value, width, and offset" + ) + value, value_type = self.emit(node.args[0]) + width, width_type = self.emit(node.args[1]) + offset, offset_type = self.emit(node.args[2]) + if _epoch_05_integer_width(value_type) is None or any( + _epoch_05_integer_width(descriptor) is None + for descriptor in (width_type, offset_type) + ): + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must be integers" + ) + name = self._new() + rendered = _render_type(value_type) + self.lines.append( + f" %{name} = ac.var.{operation} %{value}, %{width}, " + f"%{offset} : !ac.var<{rendered}>, " + f"!ac.var<{_render_type(width_type)}>, " + f"!ac.var<{_render_type(offset_type)}> -> !ac.var<{rendered}>" + ) + return self._remember(name, value_type, Unknown()) + + if operation == "bitfield_insert": + if len(node.args) != 4 or node.keywords: + raise QueueFrontendError( + "ACPY-ALU-001: bitfield_insert requires four positional operands" + ) + value, value_type = self.emit(node.args[0]) + source, source_type = self.emit(node.args[1], value_type) + width, width_type = self.emit(node.args[2]) + offset, offset_type = self.emit(node.args[3]) + if not self._types_match(value_type, source_type): + raise QueueFrontendError( + "ACPY-ALU-001: bitfield_insert value/source operands must match" + ) + if _epoch_05_integer_width(value_type) is None or any( + _epoch_05_integer_width(descriptor) is None + for descriptor in (width_type, offset_type) + ): + raise QueueFrontendError( + "ACPY-ALU-001: bitfield_insert operands must be integers" + ) + name = self._new() + rendered = _render_type(value_type) + self.lines.append( + f" %{name} = ac.var.bitfield_insert %{value}, %{source}, " + f"%{width}, %{offset} : !ac.var<{rendered}>, " + f"!ac.var<{rendered}>, !ac.var<{_render_type(width_type)}>, " + f"!ac.var<{_render_type(offset_type)}> -> !ac.var<{rendered}>" + ) + return self._remember(name, value_type, Unknown()) + + if operation in {"sext_low", "zext_low"}: + if len(node.args) != 2 or node.keywords: + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} requires value and width" + ) + value, value_type = self.emit(node.args[0]) + width, width_type = self.emit(node.args[1]) + if ( + _epoch_05_integer_width(value_type) is None + or _epoch_05_integer_width(width_type) is None + ): + raise QueueFrontendError( + f"ACPY-ALU-001: {operation} operands must be integers" + ) + name = self._new() + rendered = _render_type(value_type) + self.lines.append( + f" %{name} = ac.var.{operation} %{value}, %{width} : " + f"!ac.var<{rendered}>, !ac.var<{_render_type(width_type)}> " + f"-> !ac.var<{rendered}>" + ) + return self._remember(name, value_type, Unknown()) + + if operation == "csel": + if ( + len(node.args) != 3 + or len(node.keywords) > 1 + or any(keyword.arg != "negate_false" for keyword in node.keywords) + ): + raise QueueFrontendError( + "ACPY-ALU-001: csel requires predicate, lhs, rhs, and " + "optional negate_false" + ) + predicate, predicate_type = self.emit(node.args[0], BoolType()) + lhs, lhs_type = self.emit(node.args[1]) + rhs, rhs_type = self.emit(node.args[2], lhs_type) + negate_node = ( + node.keywords[0].value + if node.keywords + else ast.Constant(value=False) + ) + negate, negate_type = self.emit(negate_node, BoolType()) + if not self._types_match( + predicate_type, BoolType() + ) or not self._types_match(negate_type, BoolType()): + raise QueueFrontendError("ACPY-ALU-001: csel controls must be bool") + if ( + not self._types_match(lhs_type, rhs_type) + or _epoch_05_integer_width(lhs_type) is None + ): + raise QueueFrontendError( + "ACPY-ALU-001: csel selected values must match integer type" + ) + name = self._new() + self.lines.append( + f" %{name} = ac.var.csel %{predicate}, %{lhs}, %{rhs}, " + f"%{negate} : !ac.var<{_render_type(predicate_type)}>, " + f"!ac.var<{_render_type(lhs_type)}>, " + f"!ac.var<{_render_type(rhs_type)}>, " + f"!ac.var<{_render_type(negate_type)}> -> " + f"!ac.var<{_render_type(lhs_type)}>" + ) + return self._remember(name, lhs_type, Unknown()) if ( isinstance(node, ast.Call) and _decorator_name(node.func).rsplit(".", 1)[-1] == "matches" @@ -15411,9 +15720,9 @@ def name_array(names: list[str] | tuple[str, ...]) -> str: ) for input_index, input_name in enumerate(input_names): consumers.setdefault(input_name, []).append((queue, input_index)) - fanouts: dict[ - str, tuple[tuple[str, ...], tuple[tuple[QueueBinding, int], ...]] - ] = {} + fanouts: dict[str, tuple[tuple[str, ...], tuple[tuple[QueueBinding, int], ...]]] = ( + {} + ) def common_scope(scopes: list[tuple[str, ...]]) -> tuple[str, ...]: common: list[str] = [] @@ -18164,9 +18473,7 @@ def render_scope( output_signature = ( "()" if not module.outputs - else output_types - if len(module.outputs) == 1 - else f"({output_types})" + else output_types if len(module.outputs) == 1 else f"({output_types})" ) lines.append(f" }} : ({input_types}) -> {output_signature}") returned = ", ".join( @@ -18364,7 +18671,7 @@ class ModuleState: @dataclass(frozen=True, slots=True) class ModuleAssignment: - state: str + state: str | tuple[str, ...] expression: ast.expr @dataclass(frozen=True, slots=True) @@ -18625,6 +18932,67 @@ class RuleModuleTemplate: tuple(assignments), ) continue + if not states: + assignments: list[ModuleAssignment] = [] + bound_names: set[str] = set() + for statement in body[:-1]: + if ( + not isinstance(statement, ast.Assign) + or len(statement.targets) != 1 + ): + raise QueueFrontendError( + "ACPY-MODULE-001: combinational module statements must " + "assign local names" + ) + target = statement.targets[0] + if isinstance(target, ast.Name): + local_names = (target.id,) + elif isinstance(target, (ast.Tuple, ast.List)) and all( + isinstance(element, ast.Name) for element in target.elts + ): + local_names = tuple( + element.id + for element in target.elts + if isinstance(element, ast.Name) + ) + else: + raise QueueFrontendError( + "ACPY-MODULE-001: combinational assignments require " + "one local name or a result tuple" + ) + if not local_names or any( + local in bound_names or local == parameter.arg + for local in local_names + ): + raise QueueFrontendError( + "ACPY-MODULE-001: combinational module local names " + "must be unique" + ) + if len(local_names) > 1 and not ( + isinstance(statement.value, ast.Call) + and _decorator_name(statement.value.func).rsplit(".", 1)[-1] + == "divrem" + and len(local_names) == 2 + ): + raise QueueFrontendError( + "ACPY-MODULE-001: tuple assignment requires a two-result " + "divrem call" + ) + bound_names.update(local_names) + assignments.append( + ModuleAssignment( + local_names[0] if len(local_names) == 1 else local_names, + copy.deepcopy(statement.value), + ) + ) + module_types[name] = ModuleDefinition( + parameter.arg, + input_type, + outputs[0], + copy.deepcopy(body[-1].value), + assignments=tuple(assignments), + ) + continue raise QueueFrontendError( "ACPY-MODULE-001: module body requires one expression return or " "zero-initialized typed state assignments followed by return" @@ -18844,9 +19212,7 @@ def specialize_system_statements( specialized.extend(specialize_system_statements(selected)) return specialized - def specialize_rule_module( - module_name: str, call: ast.Call - ) -> tuple[ + def specialize_rule_module(module_name: str, call: ast.Call) -> tuple[ str, tuple[tuple[str, StaticValue], ...], tuple[tuple[str, ValueType], ...], @@ -19309,6 +19675,7 @@ def specialize_rule_module( expression = definition.expression if ( not definition.states + and not definition.assignments and isinstance(expression, ast.Call) and isinstance(expression.func, ast.Name) and expression.func.id in module_types @@ -19457,6 +19824,19 @@ def specialize_rule_module( invariants=invariants, helpers=helpers, ) + for assignment in definition.assignments: + if isinstance(assignment.state, tuple): + for index, local_name in enumerate(assignment.state): + local_value, local_type = emitter.emit( + ast.Subscript( + value=copy.deepcopy(assignment.expression), + slice=ast.Constant(index), + ) + ) + emitter.root_values[local_name] = (local_value, local_type) + else: + local_value, local_type = emitter.emit(assignment.expression) + emitter.root_values[assignment.state] = (local_value, local_type) value, value_type = emitter.emit(expression, output_type) if not _types_equal_in_epoch_05(value_type, output_type): raise QueueFrontendError( diff --git a/python/agentic-circuit/src/agentic_circuit/markers.py b/python/agentic-circuit/src/agentic_circuit/markers.py index 57fa9a586..ed14a73e3 100644 --- a/python/agentic-circuit/src/agentic_circuit/markers.py +++ b/python/agentic-circuit/src/agentic_circuit/markers.py @@ -8,7 +8,6 @@ from typing import Never - CAPTURE_ONLY_API = ( "scope", "map", @@ -30,6 +29,41 @@ "matches", "source", "popcount", + "udiv", + "sdiv", + "urem", + "srem", + "divrem", + "addw", + "subw", + "andw", + "orw", + "xorw", + "sll", + "srl", + "sra", + "sllw", + "srlw", + "sraw", + "smin", + "umin", + "smax", + "umax", + "mulw", + "madd", + "maddw", + "msub", + "bitfield_extract", + "bitfield_popcount", + "bitfield_clz", + "bitfield_ctz", + "bitfield_clear", + "bitfield_set", + "bitfield_reverse_bytes", + "bitfield_insert", + "sext_low", + "zext_low", + "csel", "count_leading_zeros", "count_trailing_zeros", "priority_encode", @@ -165,6 +199,177 @@ def popcount(value: object) -> Never: return _capture_time_only("popcount") +def _binary_alu(marker: str, left: object, right: object) -> Never: + _ = (left, right) + return _capture_time_only(marker) + + +def udiv(left: object, right: object) -> Never: + return _binary_alu("udiv", left, right) + + +def sdiv(left: object, right: object) -> Never: + return _binary_alu("sdiv", left, right) + + +def urem(left: object, right: object) -> Never: + return _binary_alu("urem", left, right) + + +def srem(left: object, right: object) -> Never: + return _binary_alu("srem", left, right) + + +def divrem(left: object, right: object, *, signed: object, word: object) -> Never: + _ = (left, right, signed, word) + return _capture_time_only("divrem") + + +def addw(left: object, right: object) -> Never: + return _binary_alu("addw", left, right) + + +def subw(left: object, right: object) -> Never: + return _binary_alu("subw", left, right) + + +def andw(left: object, right: object) -> Never: + return _binary_alu("andw", left, right) + + +def orw(left: object, right: object) -> Never: + return _binary_alu("orw", left, right) + + +def xorw(left: object, right: object) -> Never: + return _binary_alu("xorw", left, right) + + +def sll(left: object, right: object) -> Never: + return _binary_alu("sll", left, right) + + +def srl(left: object, right: object) -> Never: + return _binary_alu("srl", left, right) + + +def sra(left: object, right: object) -> Never: + return _binary_alu("sra", left, right) + + +def sllw(left: object, right: object) -> Never: + return _binary_alu("sllw", left, right) + + +def srlw(left: object, right: object) -> Never: + return _binary_alu("srlw", left, right) + + +def sraw(left: object, right: object) -> Never: + return _binary_alu("sraw", left, right) + + +def smin(left: object, right: object) -> Never: + return _binary_alu("smin", left, right) + + +def umin(left: object, right: object) -> Never: + return _binary_alu("umin", left, right) + + +def smax(left: object, right: object) -> Never: + return _binary_alu("smax", left, right) + + +def umax(left: object, right: object) -> Never: + return _binary_alu("umax", left, right) + + +def mulw(left: object, right: object) -> Never: + return _binary_alu("mulw", left, right) + + +def _ternary_alu(marker: str, left: object, right: object, auxiliary: object) -> Never: + _ = (left, right, auxiliary) + return _capture_time_only(marker) + + +def madd(left: object, right: object, auxiliary: object) -> Never: + return _ternary_alu("madd", left, right, auxiliary) + + +def maddw(left: object, right: object, auxiliary: object) -> Never: + return _ternary_alu("maddw", left, right, auxiliary) + + +def msub(left: object, right: object, auxiliary: object) -> Never: + return _ternary_alu("msub", left, right, auxiliary) + + +def bitfield_extract( + value: object, width: object, offset: object, *, signed: bool = False +) -> Never: + _ = (value, width, offset, signed) + return _capture_time_only("bitfield_extract") + + +def _bitfield_alu(marker: str, value: object, width: object, offset: object) -> Never: + _ = (value, width, offset) + return _capture_time_only(marker) + + +def bitfield_popcount(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_popcount", value, width, offset) + + +def bitfield_clz(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_clz", value, width, offset) + + +def bitfield_ctz(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_ctz", value, width, offset) + + +def bitfield_clear(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_clear", value, width, offset) + + +def bitfield_set(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_set", value, width, offset) + + +def bitfield_reverse_bytes(value: object, width: object, offset: object) -> Never: + return _bitfield_alu("bitfield_reverse_bytes", value, width, offset) + + +def bitfield_insert( + value: object, source: object, width: object, offset: object +) -> Never: + _ = (value, source, width, offset) + return _capture_time_only("bitfield_insert") + + +def sext_low(value: object, width: object) -> Never: + _ = (value, width) + return _capture_time_only("sext_low") + + +def zext_low(value: object, width: object) -> Never: + _ = (value, width) + return _capture_time_only("zext_low") + + +def csel( + predicate: object, + lhs: object, + rhs: object, + *, + negate_false: bool = False, +) -> Never: + _ = (predicate, lhs, rhs, negate_false) + return _capture_time_only("csel") + + def count_leading_zeros(value: object) -> Never: return _capture_time_only("count_leading_zeros") diff --git a/schemas/agentic-circuit/contracts/acir.yaml b/schemas/agentic-circuit/contracts/acir.yaml index c3e613a11..f00621cb7 100644 --- a/schemas/agentic-circuit/contracts/acir.yaml +++ b/schemas/agentic-circuit/contracts/acir.yaml @@ -51,7 +51,10 @@ operations: - ac.var.sub - ac.var.mul - ac.var.udiv + - ac.var.sdiv - ac.var.urem + - ac.var.srem + - ac.var.divrem - ac.var.and - ac.var.or - ac.var.xor @@ -80,6 +83,36 @@ operations: - ac.var.range_sub - ac.var.range_cmp - ac.var.insert + - ac.var.addw + - ac.var.subw + - ac.var.andw + - ac.var.orw + - ac.var.xorw + - ac.var.sll + - ac.var.srl + - ac.var.sra + - ac.var.sllw + - ac.var.srlw + - ac.var.sraw + - ac.var.smin + - ac.var.umin + - ac.var.smax + - ac.var.umax + - ac.var.mulw + - ac.var.madd + - ac.var.maddw + - ac.var.msub + - ac.var.bitfield_extract + - ac.var.bitfield_popcount + - ac.var.bitfield_clz + - ac.var.bitfield_ctz + - ac.var.bitfield_clear + - ac.var.bitfield_set + - ac.var.bitfield_reverse_bytes + - ac.var.bitfield_insert + - ac.var.sext_low + - ac.var.zext_low + - ac.var.csel - ac.var.decl - ac.var.read - ac.var.read_element diff --git a/simulator/gfsim/include/gfsim/alu.h b/simulator/gfsim/include/gfsim/alu.h new file mode 100644 index 000000000..4d4f809a2 --- /dev/null +++ b/simulator/gfsim/include/gfsim/alu.h @@ -0,0 +1,147 @@ +#ifndef GFSIM_ALU_H +#define GFSIM_ALU_H + +#include "gfsim/bits.h" + +#include + +namespace gfsim { + +constexpr UInt<64> signExtend32(std::uint32_t value) { + const std::uint64_t bits = value; + return UInt<64>{(value & 0x80000000U) ? bits | 0xffffffff00000000ULL : bits}; +} + +template +constexpr UInt<64> addw(UInt lhs, UInt rhs) { + return signExtend32(static_cast(lhs.value()) + + static_cast(rhs.value())); +} + +template +constexpr UInt<64> subw(UInt lhs, UInt rhs) { + return signExtend32(static_cast(lhs.value()) - + static_cast(rhs.value())); +} + +template +constexpr UInt<64> andw(UInt lhs, UInt rhs) { + return signExtend32(static_cast(lhs.value()) & + static_cast(rhs.value())); +} + +template +constexpr UInt<64> orw(UInt lhs, UInt rhs) { + return signExtend32(static_cast(lhs.value()) | + static_cast(rhs.value())); +} + +template +constexpr UInt<64> xorw(UInt lhs, UInt rhs) { + return signExtend32(static_cast(lhs.value()) ^ + static_cast(rhs.value())); +} + +constexpr UInt<64> sll(UInt<64> lhs, UInt<64> rhs) { + return UInt<64>{lhs.value() << (rhs.value() & 63U)}; +} + +constexpr UInt<64> srl(UInt<64> lhs, UInt<64> rhs) { + return UInt<64>{lhs.value() >> (rhs.value() & 63U)}; +} + +constexpr UInt<64> sra(UInt<64> lhs, UInt<64> rhs) { + return lhs.arithmeticShiftRight(UInt<64>{rhs.value() & 63U}); +} + +constexpr UInt<64> sllw(UInt<64> lhs, UInt<64> rhs) { + const std::uint32_t value = static_cast(lhs.value()); + return signExtend32(value << (rhs.value() & 31U)); +} + +constexpr UInt<64> srlw(UInt<64> lhs, UInt<64> rhs) { + const std::uint32_t value = static_cast(lhs.value()); + return signExtend32(value >> (rhs.value() & 31U)); +} + +constexpr UInt<64> sraw(UInt<64> lhs, UInt<64> rhs) { + const std::uint32_t low = static_cast(lhs.value()); + const std::uint64_t extended = + (low & 0x80000000U) + ? static_cast(low) | 0xffffffff00000000ULL + : static_cast(low); + return UInt<64>{extended}.arithmeticShiftRight(UInt<64>{rhs.value() & 31U}); +} + +constexpr UInt<64> smin(UInt<64> lhs, UInt<64> rhs) { + return signedValue(lhs) < signedValue(rhs) ? lhs : rhs; +} + +constexpr UInt<64> umin(UInt<64> lhs, UInt<64> rhs) { + return lhs.value() < rhs.value() ? lhs : rhs; +} + +constexpr UInt<64> smax(UInt<64> lhs, UInt<64> rhs) { + return signedValue(lhs) > signedValue(rhs) ? lhs : rhs; +} + +constexpr UInt<64> umax(UInt<64> lhs, UInt<64> rhs) { + return lhs.value() > rhs.value() ? lhs : rhs; +} + +constexpr UInt<64> mulw(UInt<64> lhs, UInt<64> rhs) { + return signExtend32(static_cast(lhs.value() * rhs.value())); +} + +template +constexpr UInt<64> madd(UInt<64> lhs, UInt<64> rhs, UInt aux) { + return UInt<64>{lhs.value() * rhs.value() + aux.value()}; +} + +template +constexpr UInt<64> maddw(UInt<64> lhs, UInt<64> rhs, UInt aux) { + const std::uint32_t product = + static_cast(lhs.value() * rhs.value()); + const std::uint32_t addend = static_cast(aux.value()); + return signExtend32(product + addend); +} + +template +constexpr UInt<64> msub(UInt lhs, UInt rhs, + UInt aux) { + return UInt<64>{aux.value() - lhs.value() * rhs.value()}; +} + +template +constexpr UInt<64> sextLow(UInt<64> value, UInt fieldWidth) { + const std::uint64_t width = fieldWidth.value(); + if (width == 0) + return UInt<64>{}; + if (width >= 64) + return value; + const std::uint64_t mask = (std::uint64_t{1} << width) - 1; + const std::uint64_t low = value.value() & mask; + return UInt<64>{(low & (std::uint64_t{1} << (width - 1))) ? low | ~mask + : low}; +} + +template +constexpr UInt<64> zextLow(UInt<64> value, UInt fieldWidth) { + const std::uint64_t width = fieldWidth.value(); + if (width == 0) + return UInt<64>{}; + if (width >= 64) + return value; + return UInt<64>{value.value() & ((std::uint64_t{1} << width) - 1)}; +} + +constexpr UInt<64> csel(UInt<1> predicate, UInt<64> lhs, UInt<64> rhs, + UInt<1> negateFalse) { + if (predicate.value() != 0) + return lhs; + return negateFalse.value() != 0 ? UInt<64>{0U - rhs.value()} : rhs; +} + +} // namespace gfsim + +#endif // GFSIM_ALU_H diff --git a/simulator/gfsim/include/gfsim/bitfield.h b/simulator/gfsim/include/gfsim/bitfield.h new file mode 100644 index 000000000..3cc2e93ab --- /dev/null +++ b/simulator/gfsim/include/gfsim/bitfield.h @@ -0,0 +1,139 @@ +#ifndef GFSIM_BITFIELD_H +#define GFSIM_BITFIELD_H + +#include "gfsim/bits.h" + +#include + +namespace gfsim { + +constexpr std::uint64_t bitfieldMask(std::uint64_t width) { + return width >= 64 ? ~std::uint64_t{0} + : (width == 0 ? 0 : (std::uint64_t{1} << width) - 1); +} + +constexpr std::uint64_t rotateRight64(std::uint64_t value, + std::uint64_t offset) { + offset &= 63U; + return offset == 0 ? value : (value >> offset) | (value << (64 - offset)); +} + +constexpr std::uint64_t rotateLeft64(std::uint64_t value, + std::uint64_t offset) { + offset &= 63U; + return offset == 0 ? value : (value << offset) | (value >> (64 - offset)); +} + +template +constexpr UInt<64> bitfieldExtract(UInt<64> value, UInt width, + UInt offset, + bool signedResult) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return UInt<64>{}; + const std::uint64_t result = + rotateRight64(value.value(), offset.value()) & bitfieldMask(fieldWidth); + if (!signedResult || fieldWidth == 64 || + (result & (std::uint64_t{1} << (fieldWidth - 1))) == 0) + return UInt<64>{result}; + return UInt<64>{result | ~bitfieldMask(fieldWidth)}; +} + +template +constexpr UInt<64> bitfieldPopcount(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t field = + bitfieldExtract(value, width, offset, false).value(); + return UInt<64>{static_cast(std::popcount(field) & 0x7f)}; +} + +template +constexpr UInt<64> bitfieldClz(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return UInt<64>{}; + const std::uint64_t field = + bitfieldExtract(value, width, offset, false).value(); + if (field == 0) + return UInt<64>{fieldWidth}; + std::uint64_t count = 0; + for (std::uint64_t bit = fieldWidth; + bit != 0 && ((field >> (bit - 1)) & 1U) == 0; --bit) + ++count; + return UInt<64>{count}; +} + +template +constexpr UInt<64> bitfieldCtz(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return UInt<64>{}; + const std::uint64_t field = + bitfieldExtract(value, width, offset, false).value(); + if (field == 0) + return UInt<64>{fieldWidth}; + std::uint64_t count = 0; + while (((field >> count) & 1U) == 0) + ++count; + return UInt<64>{count}; +} + +template +constexpr UInt<64> bitfieldClear(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return value; + const std::uint64_t rotated = + rotateRight64(value.value(), offset.value()) & ~bitfieldMask(fieldWidth); + return UInt<64>{rotateLeft64(rotated, offset.value())}; +} + +template +constexpr UInt<64> bitfieldSet(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return value; + const std::uint64_t rotated = + rotateRight64(value.value(), offset.value()) | bitfieldMask(fieldWidth); + return UInt<64>{rotateLeft64(rotated, offset.value())}; +} + +template +constexpr UInt<64> bitfieldReverseBytes(UInt<64> value, UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64 || (fieldWidth % 8) != 0) + return UInt<64>{}; + const std::uint64_t field = + bitfieldExtract(value, width, offset, false).value(); + std::uint64_t result = 0; + for (std::uint64_t index = 0; index < fieldWidth / 8; ++index) + result |= ((field >> (index * 8)) & 0xffU) + << ((fieldWidth / 8 - index - 1) * 8); + return UInt<64>{result}; +} + +template +constexpr UInt<64> bitfieldInsert(UInt<64> value, UInt source, + UInt width, + UInt offset) { + const std::uint64_t fieldWidth = width.value(); + if (fieldWidth == 0 || fieldWidth > 64) + return value; + std::uint64_t result = value.value(); + for (std::uint64_t bit = 0; bit < fieldWidth; ++bit) { + const std::uint64_t destination = (offset.value() + bit) & 63U; + const std::uint64_t mask = std::uint64_t{1} << destination; + result = + (result & ~mask) | (((source.value() >> bit) & 1U) != 0 ? mask : 0); + } + return UInt<64>{result}; +} + +} // namespace gfsim + +#endif // GFSIM_BITFIELD_H diff --git a/simulator/gfsim/include/gfsim/bits.h b/simulator/gfsim/include/gfsim/bits.h index 516effa74..0770b7cdd 100644 --- a/simulator/gfsim/include/gfsim/bits.h +++ b/simulator/gfsim/include/gfsim/bits.h @@ -410,6 +410,28 @@ template struct PacketTraits> { std::nullopt; }; +template +constexpr UInt signedDiv(UInt lhs, UInt rhs) { + const std::int64_t dividend = lhs.signedValue(); + const std::int64_t divisor = rhs.signedValue(); + if (divisor == 0) + return UInt{}; + if (dividend == std::numeric_limits::min() && divisor == -1) + return lhs; + return UInt{dividend / divisor}; +} + +template +constexpr UInt signedRem(UInt lhs, UInt rhs) { + const std::int64_t dividend = lhs.signedValue(); + const std::int64_t divisor = rhs.signedValue(); + if (divisor == 0) + return lhs; + if (dividend == std::numeric_limits::min() && divisor == -1) + return UInt{}; + return UInt{dividend % divisor}; +} + } // namespace gfsim #endif // GFSIM_BITS_H diff --git a/simulator/gfsim/include/gfsim/divrem.h b/simulator/gfsim/include/gfsim/divrem.h new file mode 100644 index 000000000..83676e4cd --- /dev/null +++ b/simulator/gfsim/include/gfsim/divrem.h @@ -0,0 +1,64 @@ +#ifndef GFSIM_DIVREM_H +#define GFSIM_DIVREM_H + +#include "gfsim/bits.h" + +#include +#include + +namespace gfsim { + +struct DivRemResult { + UInt<64> quotient{}; + UInt<64> remainder{}; +}; + +constexpr std::int64_t signExtend32(std::uint64_t value) { + const std::uint32_t low = static_cast(value); + return static_cast(static_cast(low)); +} + +constexpr UInt<64> signExtend32Bits(std::uint64_t value) { + return UInt<64>{static_cast(signExtend32(value))}; +} + +constexpr DivRemResult divrem(UInt<64> lhs, UInt<64> rhs, bool signed_mode, + bool word_mode) { + if (word_mode) { + if (signed_mode) { + const std::int64_t dividend = signExtend32(lhs.value()); + const std::int64_t divisor = signExtend32(rhs.value()); + if (divisor == 0) + return {UInt<64>{0}, signExtend32Bits(lhs.value())}; + if (dividend == std::numeric_limits::min() && divisor == -1) + return {signExtend32Bits(0x80000000ULL), UInt<64>{0}}; + return {signExtend32Bits(static_cast(dividend / divisor)), + signExtend32Bits(static_cast(dividend % divisor))}; + } + const std::uint64_t dividend = lhs.value() & 0xffffffffULL; + const std::uint64_t divisor = rhs.value() & 0xffffffffULL; + if (divisor == 0) + return {UInt<64>{0}, signExtend32Bits(dividend)}; + return {signExtend32Bits(static_cast(dividend / divisor)), + signExtend32Bits(static_cast(dividend % divisor))}; + } + + if (signed_mode) { + const std::int64_t dividend = lhs.signedValue(); + const std::int64_t divisor = rhs.signedValue(); + if (divisor == 0) + return {UInt<64>{0}, lhs}; + if (dividend == std::numeric_limits::min() && divisor == -1) + return {lhs, UInt<64>{0}}; + return {UInt<64>{dividend / divisor}, UInt<64>{dividend % divisor}}; + } + const std::uint64_t dividend = lhs.value(); + const std::uint64_t divisor = rhs.value(); + if (divisor == 0) + return {UInt<64>{0}, lhs}; + return {UInt<64>{dividend / divisor}, UInt<64>{dividend % divisor}}; +} + +} // namespace gfsim + +#endif // GFSIM_DIVREM_H diff --git a/tests/cpp/agentic-circuit/CodeGen/QueueGraphPlanTest.cpp b/tests/cpp/agentic-circuit/CodeGen/QueueGraphPlanTest.cpp index 94559051e..b89ca60c1 100644 --- a/tests/cpp/agentic-circuit/CodeGen/QueueGraphPlanTest.cpp +++ b/tests/cpp/agentic-circuit/CodeGen/QueueGraphPlanTest.cpp @@ -3175,6 +3175,99 @@ TEST(QueueGraphPlanTest, NativeGeneratorConsumesOnlyExtractedPlan) { std::string::npos); } +TEST(QueueGraphPlanTest, EmitsAluSemanticExpressionsToGfsimPrimitives) { + QueueGraphPlan plan; + plan.system = "alu_semantics"; + plan.queues = { + {"input", "i64", "/", 1, 1}, {"addw_out", "i64", "/", 1, 1}, + {"sraw_out", "i64", "/", 1, 1}, {"maddw_out", "i64", "/", 1, 1}, + {"extract_out", "i64", "/", 1, 1}, {"insert_out", "i64", "/", 1, 1}, + {"selected_out", "i64", "/", 1, 1}}; + plan.blocks.push_back({"source", "input", "/", {}, {"input"}, {1}, {1}}); + QueueBlockPlan transform{"transform", + "alu_outputs", + "/", + {"input"}, + {"addw_out", "sraw_out", "maddw_out", "extract_out", + "insert_out", "selected_out"}, + {1, 1, 1, 1, 1, 1}, + {1, 1, 1, 1, 1, 1}}; + transform.expressions = { + {"one", "constant", "i64", {}, "", "", "1 : i64"}, + {"two", "constant", "i64", {}, "", "", "2 : i64"}, + {"width", "constant", "i7", {}, "", "", "8 : i7"}, + {"offset", "constant", "i6", {}, "", "", "4 : i6"}, + {"predicate", "constant", "i1", {}, "", "", "true"}, + {"negate", "constant", "i1", {}, "", "", "false"}, + {"addw", "addw", "i64", {"item", "one"}}, + {"sraw", "sraw", "i64", {"item", "one"}}, + {"maddw", "maddw", "i64", {"item", "one", "two"}}, + {"extract", + "bitfield_extract", + "i64", + {"item", "width", "offset"}, + "", + "signed"}, + {"insert", "bitfield_insert", "i64", {"item", "two", "width", "offset"}}, + {"selected", "csel", "i64", {"predicate", "item", "two", "negate"}}, + }; + transform.yields = {"addw", "sraw", "maddw", "extract", "insert", "selected"}; + plan.blocks.push_back(std::move(transform)); + plan.blocks.push_back({"sink", "sink_0", "/", {"addw_out"}, {}}); + plan.blocks.push_back({"sink", "sink_1", "/", {"sraw_out"}, {}}); + plan.blocks.push_back({"sink", "sink_2", "/", {"maddw_out"}, {}}); + plan.blocks.push_back({"sink", "sink_3", "/", {"extract_out"}, {}}); + plan.blocks.push_back({"sink", "sink_4", "/", {"insert_out"}, {}}); + plan.blocks.push_back({"sink", "sink_5", "/", {"selected_out"}, {}}); + + auto generated = generateQueueGraphCpp(plan); + ASSERT_TRUE(bool(generated)) << llvm::toString(generated.takeError()); + llvm::StringRef source(*generated); + EXPECT_NE(source.find("gfsim::addw("), std::string::npos); + EXPECT_NE(source.find("gfsim::sraw("), std::string::npos); + EXPECT_NE(source.find("gfsim::maddw("), std::string::npos); + EXPECT_NE(source.find("gfsim::bitfieldExtract("), std::string::npos); + EXPECT_NE(source.find("gfsim::bitfieldInsert("), std::string::npos); + EXPECT_NE(source.find("gfsim::csel("), std::string::npos); + expectCppCompiles(*generated); +} + +TEST(QueueGraphPlanTest, EmitsOneSharedPycDividerForQuotientAndRemainder) { + QueueGraphPlan plan; + plan.system = "shared_divrem"; + plan.queues = {{"input", "i64", "/", 1, 1}, + {"quotient", "i64", "/", 1, 1}, + {"remainder", "i64", "/", 1, 1}}; + plan.blocks.push_back({"source", "input", "/", {}, {"input"}, {1}, {1}}); + QueueBlockPlan transform{ + "transform", "divide", "/", {"input"}, {"quotient", "remainder"}, + {1, 1}, {1, 1}}; + transform.expressions = { + {"rhs", "constant", "i64", {}, "", "", "3 : i64"}, + {"signed", "constant", "i1", {}, "", "", "true"}, + {"word", "constant", "i1", {}, "", "", "false"}, + {"quotient_value", + "divrem_quotient", + "i64", + {"item", "rhs", "signed", "word"}}, + {"remainder_value", + "divrem_remainder", + "i64", + {"item", "rhs", "signed", "word"}}, + }; + transform.yields = {"quotient_value", "remainder_value"}; + plan.blocks.push_back(std::move(transform)); + plan.blocks.push_back({"sink", "quotient_sink", "/", {"quotient"}, {}}); + plan.blocks.push_back({"sink", "remainder_sink", "/", {"remainder"}, {}}); + + auto generated = generateQueueGraphPyc(plan); + ASSERT_TRUE(bool(generated)) << llvm::toString(generated.takeError()); + llvm::StringRef source(*generated); + EXPECT_EQ(source.count("pyc.divrem"), 1u); + EXPECT_NE(source.find(" : (i64, i64, i1, i1) -> (i64, i64)"), + std::string::npos); +} + TEST(QueueGraphPlanTest, EmitsClosedOpaqueRuntimeAbiBundle) { mlir::MLIRContext context; context.loadDialect(); diff --git a/tests/cpp/agentic-circuit/Dialect/ACIR/OpsTest.cpp b/tests/cpp/agentic-circuit/Dialect/ACIR/OpsTest.cpp index 06c704090..d0e23a946 100644 --- a/tests/cpp/agentic-circuit/Dialect/ACIR/OpsTest.cpp +++ b/tests/cpp/agentic-circuit/Dialect/ACIR/OpsTest.cpp @@ -40,17 +40,18 @@ TEST(ACIROpsTest, SemanticPrimitiveWidthsAcceptOneTo64AndReject65To130) { const unsigned countWidth = acir::primitiveCountWidth(width); std::string source; llvm::raw_string_ostream stream(source); - stream << "builtin.module attributes {ac.contract_epoch = \"0.5\"} {\n" - << " %value = \"builtin.unrealized_conversion_cast\"() : () -> " - << "!ac.var\n" - << " %index, %valid = ac.var.priority_encode %value order \"low\" : " - << "!ac.var -> !ac.var, !ac.var\n" - << " %count = ac.var.popcount %value : !ac.var -> !ac.var\n" - << " %zeros = ac.var.count_zeros %value direction \"leading\" : " - << "!ac.var -> !ac.var\n" - << "}\n"; + stream + << "builtin.module attributes {ac.contract_epoch = \"0.5\"} {\n" + << " %value = \"builtin.unrealized_conversion_cast\"() : () -> " + << "!ac.var\n" + << " %index, %valid = ac.var.priority_encode %value order \"low\" : " + << "!ac.var -> !ac.var, !ac.var\n" + << " %count = ac.var.popcount %value : !ac.var -> !ac.var\n" + << " %zeros = ac.var.count_zeros %value direction \"leading\" : " + << "!ac.var -> !ac.var\n" + << "}\n"; auto module = mlir::parseSourceString(source, &context); EXPECT_EQ(static_cast(module), width <= 64) << "width " << width; } @@ -298,6 +299,7 @@ TEST(ACIROpsTest, RegistryContainsExactQueueVarOperations) { "ac.table.yield", "ac.slot", "ac.slot.get", + "ac.slot.propose_release", "ac.slot.release", "ac.slot.yield", "ac.reorder", @@ -310,17 +312,28 @@ TEST(ACIROpsTest, RegistryContainsExactQueueVarOperations) { "ac.protocol", "ac.queue", "ac.var.add", + "ac.var.addw", "ac.var.and", + "ac.var.andw", "ac.var.array", "ac.var.record", "ac.var.assign", "ac.var.assign_element", + "ac.var.bitfield_clear", + "ac.var.bitfield_clz", + "ac.var.bitfield_ctz", + "ac.var.bitfield_extract", + "ac.var.bitfield_insert", + "ac.var.bitfield_popcount", + "ac.var.bitfield_reverse_bytes", + "ac.var.bitfield_set", "ac.var.choose", "ac.var.choose.yield", "ac.var.concat", "ac.var.constant", "ac.var.count_zeros", "ac.var.cmp", + "ac.var.csel", "ac.var.decl", "ac.var.dynamic_element", "ac.var.element", @@ -333,11 +346,19 @@ TEST(ACIROpsTest, RegistryContainsExactQueueVarOperations) { "ac.var.invariant.yield", "ac.var.match", "ac.var.match.yield", + "ac.var.madd", + "ac.var.maddw", "ac.var.mul", + "ac.var.mulw", + "ac.var.msub", "ac.var.udiv", + "ac.var.sdiv", "ac.var.urem", + "ac.var.srem", + "ac.var.divrem", "ac.var.not", "ac.var.or", + "ac.var.orw", "ac.var.popcount", "ac.var.priority_encode", "ac.var.range_add", @@ -351,14 +372,28 @@ TEST(ACIROpsTest, RegistryContainsExactQueueVarOperations) { "ac.var.read", "ac.var.read_element", "ac.var.select", + "ac.var.sext_low", "ac.var.shl", "ac.var.shr", + "ac.var.sll", + "ac.var.sllw", + "ac.var.smax", + "ac.var.smin", + "ac.var.sra", + "ac.var.sraw", + "ac.var.srl", + "ac.var.srlw", "ac.var.matches", "ac.var.sub", + "ac.var.subw", "ac.var.tuple", "ac.var.xor", + "ac.var.xorw", "ac.var.with", "ac.var.with_element", + "ac.var.umax", + "ac.var.umin", + "ac.var.zext_low", "ac.require", "ac.return", "ac.resource", @@ -479,8 +514,8 @@ TEST(ACIROpsTest, TaskSixRegistryDeltaIsExactlyEightGraphOperations) { mlir::MLIRContext context; context.loadDialect(); const std::array names = { - "ac.system", "ac.module", "ac.module.extern", "ac.instance", - "ac.array", "ac.instances", "ac.view", "ac.return", + "ac.system", "ac.module", "ac.module.extern", "ac.instance", + "ac.array", "ac.instances", "ac.view", "ac.return", }; for (llvm::StringLiteral name : names) EXPECT_TRUE(mlir::OperationName(name, &context).isRegistered()) @@ -706,11 +741,10 @@ TEST(ACIROpsTest, RuntimeAndQueueVarRegistryIsExact) { mlir::MLIRContext context; context.loadDialect(); const std::array names = { - "ac.process", "ac.try_send", "ac.try_recv", - "ac.schedule", "ac.wait_until", "ac.wait_for", - "ac.await_event", "ac.yield_sim", "ac.require", "ac.ensure", - "ac.assert", "ac.probe", "ac.stat", "ac.stat.add", - "ac.instrumentation", + "ac.process", "ac.try_send", "ac.try_recv", "ac.schedule", + "ac.wait_until", "ac.wait_for", "ac.await_event", "ac.yield_sim", + "ac.require", "ac.ensure", "ac.assert", "ac.probe", + "ac.stat", "ac.stat.add", "ac.instrumentation", }; for (llvm::StringLiteral name : names) EXPECT_TRUE(mlir::OperationName(name, &context).isRegistered()) @@ -830,10 +864,9 @@ TEST(ACIROpsTest, RuntimeAndQueueVarRegistryIsExact) { for (llvm::StringLiteral name : queueVarNames) EXPECT_TRUE(mlir::OperationName(name, &context).isRegistered()) << name.str(); - EXPECT_EQ(context.getRegisteredOperationsByDialect("ac").size(), 151u); + EXPECT_EQ(context.getRegisteredOperationsByDialect("ac").size(), 185u); } - TEST(ACIROpsTest, LargeArrayVerificationIsDeterministic) { mlir::MLIRContext context; context.loadDialect(); @@ -1294,7 +1327,6 @@ TEST(ACIROpsTest, TaskEightOwnersParticipateInSaturatedArrayBudget) { std::string::npos); } - TEST(ACIROpsTest, StaticContractsUseFreezePhaseModuleEffects) { mlir::MLIRContext context; context.loadDialect(); diff --git a/tests/cpp/agentic-circuit/gfsim/AluPrimitivesTest.cpp b/tests/cpp/agentic-circuit/gfsim/AluPrimitivesTest.cpp new file mode 100644 index 000000000..11cf4bfe9 --- /dev/null +++ b/tests/cpp/agentic-circuit/gfsim/AluPrimitivesTest.cpp @@ -0,0 +1,82 @@ +#include "gfsim/alu.h" +#include "gfsim/bitfield.h" + +#include "gtest/gtest.h" + +#include + +namespace gfsim { +namespace { + +TEST(AluPrimitivesTest, WordArithmeticLogicAndShiftsWrapArchitecturally) { + EXPECT_EQ(addw(UInt<64>{0x7fffffff}, UInt<64>{1}).value(), + 0xffffffff80000000ULL); + EXPECT_EQ(subw(UInt<64>{0}, UInt<64>{1}).value(), 0xffffffffffffffffULL); + EXPECT_EQ(andw(UInt<64>{0xffffffff}, UInt<64>{0x80000001}).value(), + 0xffffffff80000001ULL); + EXPECT_EQ(orw(UInt<64>{0x80000000}, UInt<64>{0}).value(), + 0xffffffff80000000ULL); + EXPECT_EQ(xorw(UInt<64>{0x80000000}, UInt<64>{0xffffffff}).value(), + 0x7fffffffULL); + + EXPECT_EQ(sll(UInt<64>{1}, UInt<64>{64}).value(), 1ULL); + EXPECT_EQ(srl(UInt<64>{0x80}, UInt<64>{67}).value(), 0x10ULL); + EXPECT_EQ(sra(UInt<64>{0x8000000000000000ULL}, UInt<64>{63}).value(), + 0xffffffffffffffffULL); + EXPECT_EQ(sllw(UInt<64>{1}, UInt<64>{32}).value(), 1ULL); + EXPECT_EQ(srlw(UInt<64>{0xffffffff}, UInt<64>{4}).value(), 0x0fffffffULL); + EXPECT_EQ(sraw(UInt<64>{0x80000000}, UInt<64>{4}).value(), + 0xfffffffff8000000ULL); +} + +TEST(AluPrimitivesTest, SignedUnsignedCompareAndMultiplyWrap) { + EXPECT_EQ(smin(UInt<64>{0x8000000000000000ULL}, UInt<64>{0}).value(), + 0x8000000000000000ULL); + EXPECT_EQ(smax(UInt<64>{0x8000000000000000ULL}, UInt<64>{0}).value(), 0ULL); + EXPECT_EQ(umin(UInt<64>{1}, UInt<64>{~std::uint64_t{0}}).value(), 1ULL); + EXPECT_EQ(umax(UInt<64>{1}, UInt<64>{~std::uint64_t{0}}).value(), + ~std::uint64_t{0}); + EXPECT_EQ(mulw(UInt<64>{0xffffffff}, UInt<64>{2}).value(), + 0xfffffffffffffffeULL); + EXPECT_EQ(madd(UInt<64>{7}, UInt<64>{5}, UInt<64>{3}).value(), 38ULL); + EXPECT_EQ(maddw(UInt<64>{0xffffffff}, UInt<64>{2}, UInt<64>{1}).value(), + 0xffffffffffffffffULL); + EXPECT_EQ(msub(UInt<64>{7}, UInt<64>{5}, UInt<64>{3}).value(), + 0xffffffffffffffe0ULL); +} + +TEST(AluPrimitivesTest, WrappingBitfieldsAndZeroScansHaveDefinedEdges) { + constexpr UInt<64> wrapped = + bitfieldInsert(UInt<64>{0}, UInt<64>{0xab}, UInt<7>{8}, UInt<6>{60}); + static_assert(wrapped.value() == 0xb000000000000000ULL + 0xAULL); + EXPECT_EQ(wrapped.value(), 0xb000000000000000ULL + 0xAULL); + EXPECT_EQ(bitfieldExtract(UInt<64>{0xf000000000000001ULL}, UInt<7>{8}, + UInt<6>{60}, false) + .value(), + 0x1fULL); + EXPECT_EQ(bitfieldClz(UInt<64>{0}, UInt<7>{8}, UInt<6>{0}).value(), 8ULL); + EXPECT_EQ(bitfieldCtz(UInt<64>{0}, UInt<7>{8}, UInt<6>{0}).value(), 8ULL); + EXPECT_EQ( + bitfieldReverseBytes(UInt<64>{0xffff}, UInt<7>{7}, UInt<6>{0}).value(), + 0ULL); + EXPECT_EQ(bitfieldClear(UInt<64>{0xffff}, UInt<7>{8}, UInt<6>{4}).value(), + 0xf00fULL); + EXPECT_EQ(bitfieldSet(UInt<64>{0}, UInt<7>{8}, UInt<6>{4}).value(), 0xff0ULL); +} + +TEST(AluPrimitivesTest, SelectExtensionsAndNegatedFalsePath) { + EXPECT_EQ( + csel(UInt<1>{1}, UInt<64>{0x1111}, UInt<64>{0x2222}, UInt<1>{1}).value(), + 0x1111ULL); + EXPECT_EQ( + csel(UInt<1>{0}, UInt<64>{0x1111}, UInt<64>{0x2222}, UInt<1>{0}).value(), + 0x2222ULL); + EXPECT_EQ( + csel(UInt<1>{0}, UInt<64>{0x1111}, UInt<64>{0x2222}, UInt<1>{1}).value(), + 0xffffffffffffdddeULL); + EXPECT_EQ(sextLow(UInt<64>{0x80}, UInt<7>{8}).value(), 0xffffffffffffff80ULL); + EXPECT_EQ(zextLow(UInt<64>{0x180}, UInt<7>{8}).value(), 0x80ULL); +} + +} // namespace +} // namespace gfsim diff --git a/tests/cpp/agentic-circuit/gfsim/CMakeLists.txt b/tests/cpp/agentic-circuit/gfsim/CMakeLists.txt index b8a003240..d607930e5 100644 --- a/tests/cpp/agentic-circuit/gfsim/CMakeLists.txt +++ b/tests/cpp/agentic-circuit/gfsim/CMakeLists.txt @@ -1,5 +1,6 @@ find_package(GTest CONFIG REQUIRED) add_executable(GfsimTests + AluPrimitivesTest.cpp BitsTest.cpp QueueBlocksTest.cpp ShowcaseTest.cpp diff --git a/tests/mlir/agentic-circuit/ACIR/alu-primitives-invalid.mlir b/tests/mlir/agentic-circuit/ACIR/alu-primitives-invalid.mlir new file mode 100644 index 000000000..fd5603186 --- /dev/null +++ b/tests/mlir/agentic-circuit/ACIR/alu-primitives-invalid.mlir @@ -0,0 +1,43 @@ +// RUN: %split_file %s %t +// RUN: %not %acir_opt %t/BINARY-WIDTH.mlir 2>&1 | %FileCheck %s --check-prefix=BINARY-WIDTH +// RUN: %not %acir_opt %t/BITFIELD-WIDTH.mlir 2>&1 | %FileCheck %s --check-prefix=BITFIELD-WIDTH +// RUN: %not %acir_opt %t/BITFIELD-TYPES.mlir 2>&1 | %FileCheck %s --check-prefix=BITFIELD-TYPES +// RUN: %not %acir_opt %t/CSEL-PREDICATE.mlir 2>&1 | %FileCheck %s --check-prefix=CSEL-PREDICATE + +// BINARY-WIDTH: error: 'ac.var.addw' op operand width must be 64 +// BITFIELD-WIDTH: error: 'ac.var.bitfield_clz' op operand width must be 7 +// BITFIELD-TYPES: error: 'ac.var.bitfield_insert' op value, source, and result must have one identical Var type +// CSEL-PREDICATE: error: 'ac.var.csel' op predicate and negate_false must be !ac.var + +//--- BINARY-WIDTH.mlir +builtin.module attributes {ac.contract_epoch = "0.5"} { + %lhs = ac.var.constant 1 : i32 as !ac.var + %rhs = ac.var.constant 2 : i32 as !ac.var + %bad = ac.var.addw %lhs, %rhs : !ac.var -> !ac.var +} + +//--- BITFIELD-WIDTH.mlir +builtin.module attributes {ac.contract_epoch = "0.5"} { + %value = ac.var.constant 1 : i64 as !ac.var + %width = ac.var.constant 8 : i8 as !ac.var + %offset = ac.var.constant 0 : i6 as !ac.var + %bad = ac.var.bitfield_clz %value, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var +} + +//--- BITFIELD-TYPES.mlir +builtin.module attributes {ac.contract_epoch = "0.5"} { + %value = ac.var.constant 1 : i64 as !ac.var + %source = ac.var.constant 2 : i32 as !ac.var + %width = ac.var.constant 8 : i7 as !ac.var + %offset = ac.var.constant 0 : i6 as !ac.var + %bad = ac.var.bitfield_insert %value, %source, %width, %offset : !ac.var, !ac.var, !ac.var, !ac.var -> !ac.var +} + +//--- CSEL-PREDICATE.mlir +builtin.module attributes {ac.contract_epoch = "0.5"} { + %predicate = ac.var.constant 1 : i8 as !ac.var + %lhs = ac.var.constant 2 : i64 as !ac.var + %rhs = ac.var.constant 3 : i64 as !ac.var + %negate = ac.var.constant 0 : i1 as !ac.var + %bad = ac.var.csel %predicate, %lhs, %rhs, %negate : !ac.var, !ac.var, !ac.var, !ac.var -> !ac.var +} diff --git a/tests/mlir/agentic-circuit/ACIR/alu-primitives.mlir b/tests/mlir/agentic-circuit/ACIR/alu-primitives.mlir new file mode 100644 index 000000000..749e0eaf5 --- /dev/null +++ b/tests/mlir/agentic-circuit/ACIR/alu-primitives.mlir @@ -0,0 +1,73 @@ +// RUN: %acir_opt %s | %FileCheck %s + +builtin.module attributes {ac.contract_epoch = "0.5"} { + %lhs = ac.var.constant 0x8000000000000001 : i64 as !ac.var + %rhs = ac.var.constant 3 : i64 as !ac.var + %aux = ac.var.constant 5 : i64 as !ac.var + %width = ac.var.constant 8 : i7 as !ac.var + %offset = ac.var.constant 4 : i6 as !ac.var + %pred = ac.var.constant 1 : i1 as !ac.var + %negate = ac.var.constant 0 : i1 as !ac.var + %addw = ac.var.addw %lhs, %rhs : !ac.var -> !ac.var + %subw = ac.var.subw %lhs, %rhs : !ac.var -> !ac.var + %andw = ac.var.andw %lhs, %rhs : !ac.var -> !ac.var + %orw = ac.var.orw %lhs, %rhs : !ac.var -> !ac.var + %xorw = ac.var.xorw %lhs, %rhs : !ac.var -> !ac.var + %sll = ac.var.sll %lhs, %rhs : !ac.var -> !ac.var + %srl = ac.var.srl %lhs, %rhs : !ac.var -> !ac.var + %sra = ac.var.sra %lhs, %rhs : !ac.var -> !ac.var + %sllw = ac.var.sllw %lhs, %rhs : !ac.var -> !ac.var + %srlw = ac.var.srlw %lhs, %rhs : !ac.var -> !ac.var + %sraw = ac.var.sraw %lhs, %rhs : !ac.var -> !ac.var + %smin = ac.var.smin %lhs, %rhs : !ac.var -> !ac.var + %umin = ac.var.umin %lhs, %rhs : !ac.var -> !ac.var + %smax = ac.var.smax %lhs, %rhs : !ac.var -> !ac.var + %umax = ac.var.umax %lhs, %rhs : !ac.var -> !ac.var + %mulw = ac.var.mulw %lhs, %rhs : !ac.var -> !ac.var + %madd = ac.var.madd %lhs, %rhs, %aux : !ac.var -> !ac.var + %maddw = ac.var.maddw %lhs, %rhs, %aux : !ac.var -> !ac.var + %msub = ac.var.msub %lhs, %rhs, %aux : !ac.var -> !ac.var + %bxs = ac.var.bitfield_extract %lhs, %width, %offset signed_mode true : !ac.var, !ac.var, !ac.var -> !ac.var + %bxu = ac.var.bitfield_extract %lhs, %width, %offset signed_mode false : !ac.var, !ac.var, !ac.var -> !ac.var + %bcnt = ac.var.bitfield_popcount %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %clz = ac.var.bitfield_clz %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %ctz = ac.var.bitfield_ctz %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %bic = ac.var.bitfield_clear %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %bis = ac.var.bitfield_set %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %rev = ac.var.bitfield_reverse_bytes %lhs, %width, %offset : !ac.var, !ac.var, !ac.var -> !ac.var + %bfi = ac.var.bitfield_insert %lhs, %rhs, %width, %offset : !ac.var, !ac.var, !ac.var, !ac.var -> !ac.var + %sext = ac.var.sext_low %lhs, %width : !ac.var, !ac.var -> !ac.var + %zext = ac.var.zext_low %lhs, %width : !ac.var, !ac.var -> !ac.var + %csel = ac.var.csel %pred, %lhs, %rhs, %negate : !ac.var, !ac.var, !ac.var, !ac.var -> !ac.var +} + +// CHECK: ac.var.addw +// CHECK: ac.var.subw +// CHECK: ac.var.andw +// CHECK: ac.var.orw +// CHECK: ac.var.xorw +// CHECK: ac.var.sll +// CHECK: ac.var.srl +// CHECK: ac.var.sra +// CHECK: ac.var.sllw +// CHECK: ac.var.srlw +// CHECK: ac.var.sraw +// CHECK: ac.var.smin +// CHECK: ac.var.umin +// CHECK: ac.var.smax +// CHECK: ac.var.umax +// CHECK: ac.var.mulw +// CHECK: ac.var.madd +// CHECK: ac.var.maddw +// CHECK: ac.var.msub +// CHECK: ac.var.bitfield_extract +// CHECK: ac.var.bitfield_popcount +// CHECK: ac.var.bitfield_clz +// CHECK: ac.var.bitfield_ctz +// CHECK: ac.var.bitfield_clear +// CHECK: ac.var.bitfield_set +// CHECK: ac.var.bitfield_reverse_bytes +// CHECK: ac.var.bitfield_insert +// CHECK: ac.var.sext_low +// CHECK: ac.var.zext_low +// CHECK: ac.var.csel diff --git a/tests/python/agentic-circuit/python_frontend/test_public_api.py b/tests/python/agentic-circuit/python_frontend/test_public_api.py index d41f0ec8d..0c6dd041b 100644 --- a/tests/python/agentic-circuit/python_frontend/test_public_api.py +++ b/tests/python/agentic-circuit/python_frontend/test_public_api.py @@ -27,6 +27,41 @@ "count_leading_zeros", "count_trailing_zeros", "popcount", + "udiv", + "sdiv", + "urem", + "srem", + "divrem", + "addw", + "subw", + "andw", + "orw", + "xorw", + "sll", + "srl", + "sra", + "sllw", + "srlw", + "sraw", + "smin", + "umin", + "smax", + "umax", + "mulw", + "madd", + "maddw", + "msub", + "bitfield_extract", + "bitfield_popcount", + "bitfield_clz", + "bitfield_ctz", + "bitfield_clear", + "bitfield_set", + "bitfield_reverse_bytes", + "bitfield_insert", + "sext_low", + "zext_low", + "csel", "priority_encode", "onehot_encode", "onehot_enum", diff --git a/tests/python/agentic-circuit/python_frontend/test_queue_frontend.py b/tests/python/agentic-circuit/python_frontend/test_queue_frontend.py index 369567441..ebf3bc18f 100644 --- a/tests/python/agentic-circuit/python_frontend/test_queue_frontend.py +++ b/tests/python/agentic-circuit/python_frontend/test_queue_frontend.py @@ -139,6 +139,70 @@ def pipeline() -> None: sink(output_queue) """ +ALU_PRIMITIVE_SOURCE = """ +import agentic_circuit as ac + +@ac.struct +class Item: + lhs: ac.u64 + rhs: ac.u64 + aux: ac.u64 + width: ac.u7 + offset: ac.u6 + predicate: ac.u1 + addw_result: ac.u64 + sraw_result: ac.u64 + maddw_result: ac.u64 + extract_result: ac.u64 + insert_result: ac.u64 + csel_result: ac.u64 + +@ac.system +def pipeline() -> None: + incoming = ac.source(Item) + outgoing = incoming.apply( + lambda item: item.with_fields( + addw_result=ac.addw(item.lhs, item.rhs), + sraw_result=ac.sraw(item.lhs, item.rhs), + maddw_result=ac.maddw(item.lhs, item.rhs, item.aux), + extract_result=ac.bitfield_extract( + item.lhs, item.width, item.offset, signed=True + ), + insert_result=ac.bitfield_insert( + item.lhs, item.rhs, item.width, item.offset + ), + csel_result=ac.csel( + item.predicate, item.lhs, item.rhs, negate_false=True + ), + ) + ) + ac.sink(outgoing) +""" + +DIVREM_SOURCE = """ +import agentic_circuit as ac + +@ac.struct +class DivPacket: + lhs: ac.u64 + rhs: ac.u64 + signed: ac.u1 + word: ac.u1 + quotient: ac.u64 + remainder: ac.u64 + +@ac.module +def divide(item: DivPacket) -> DivPacket: + quotient, remainder = ac.divrem( + item.lhs, item.rhs, signed=item.signed, word=item.word + ) + return item.with_fields(quotient=quotient, remainder=remainder) + +@ac.system +def pipeline(item: DivPacket) -> DivPacket: + return divide(item) +""" + STRUCT_SOURCE = """ from agentic_circuit import sink, source, struct, system @@ -2660,6 +2724,53 @@ def test_zero_counts_lower_to_one_parameterized_var_operation(self) -> None: self.assertIn('direction "trailing" : !ac.var -> !ac.var', lowered) self.assertEqual(lowered.count("ac.var.count_zeros"), 2) + def test_alu_semantic_primitives_lower_to_typed_var_operations(self) -> None: + from agentic_circuit._queue_frontend import lower_queue_source + + lowered = lower_queue_source(ALU_PRIMITIVE_SOURCE, "pipeline") + for operation in ( + "addw", + "sraw", + "maddw", + "bitfield_extract", + "bitfield_insert", + "csel", + ): + with self.subTest(operation=operation): + self.assertIn(f"ac.var.{operation}", lowered) + self.assertIn("signed_mode true", lowered) + self.assertIn("ac.var.csel", lowered) + self.assertIn("ac.var.constant true as !ac.var", lowered) + + def test_alu_semantic_primitives_reject_invalid_operands_and_modes(self) -> None: + from agentic_circuit._queue_frontend import ( + QueueFrontendError, + lower_queue_source, + ) + + with self.assertRaisesRegex(QueueFrontendError, "addw operands must match"): + lower_queue_source( + ALU_PRIMITIVE_SOURCE.replace( + "ac.addw(item.lhs, item.rhs)", + "ac.addw(item.lhs, item.width)", + ), + "pipeline", + ) + with self.assertRaisesRegex( + QueueFrontendError, "bitfield_extract signed must be static bool" + ): + lower_queue_source( + ALU_PRIMITIVE_SOURCE.replace("signed=True", "signed=item.predicate"), + "pipeline", + ) + + def test_divrem_lowers_once_with_two_reused_results(self) -> None: + from agentic_circuit._queue_frontend import lower_queue_source + + lowered = lower_queue_source(DIVREM_SOURCE, "pipeline") + self.assertEqual(lowered.count("ac.var.divrem"), 1) + self.assertRegex(lowered, r"%v\d+, %v\d+ = ac\.var\.divrem") + def test_verification_expect_is_non_consuming_and_role_explicit(self) -> None: from agentic_circuit._queue_frontend import ( QueueFrontendError, @@ -7748,9 +7859,9 @@ def test_state_proposals_capture_the_local_version_at_source_position(self) -> N lowered = lower_queue_source(SERIAL_SOURCE_ORDER_SOURCE, "serial_source_order") field_values = { - line.split('field "', 1)[1].split('"', 1)[0]: line.split("%", 1)[1].split( - " ", 1 - )[0] + line.split('field "', 1)[1] + .split('"', 1)[0]: line.split("%", 1)[1] + .split(" ", 1)[0] for line in lowered.splitlines() if "ac.var.get %item field" in line } @@ -7789,9 +7900,9 @@ def test_branch_guards_capture_the_local_version_at_branch_entry(self) -> None: lowered = lower_queue_source(SERIAL_GUARD_REBIND_SOURCE, "serial_guard_rebind") field_values = { - line.split('field "', 1)[1].split('"', 1)[0]: line.split("%", 1)[1].split( - " ", 1 - )[0] + line.split('field "', 1)[1] + .split('"', 1)[0]: line.split("%", 1)[1] + .split(" ", 1)[0] for line in lowered.splitlines() if "ac.var.get %item field" in line } @@ -7811,9 +7922,9 @@ def test_nested_branch_path_captures_each_condition_before_rebinding(self) -> No NESTED_BRANCH_GUARD_REBIND_SOURCE, "nested_branch_guard_rebind" ) field_values = { - line.split('field "', 1)[1].split('"', 1)[0]: line.split("%", 1)[1].split( - " ", 1 - )[0] + line.split('field "', 1)[1] + .split('"', 1)[0]: line.split("%", 1)[1] + .split(" ", 1)[0] for line in lowered.splitlines() if "ac.var.get %item field" in line } @@ -7886,9 +7997,9 @@ def test_early_return_guards_capture_each_source_local_version(self) -> None: SERIAL_EARLY_GUARD_REBIND_SOURCE, "serial_early_guard_rebind" ) field_values = { - line.split('field "', 1)[1].split('"', 1)[0]: line.split("%", 1)[1].split( - " ", 1 - )[0] + line.split('field "', 1)[1] + .split('"', 1)[0]: line.split("%", 1)[1] + .split(" ", 1)[0] for line in lowered.splitlines() if "ac.var.get %item field" in line } diff --git a/tests/python/agentic-circuit/tools/test_pyc_verilog_backend.py b/tests/python/agentic-circuit/tools/test_pyc_verilog_backend.py index f2849ac79..53e961bd1 100644 --- a/tests/python/agentic-circuit/tools/test_pyc_verilog_backend.py +++ b/tests/python/agentic-circuit/tools/test_pyc_verilog_backend.py @@ -36,6 +36,27 @@ def load_tool(): } """ +PYC_DIVREM_PACKET = """ +module attributes {pyc.top = @div_hold} { + func.func @div_hold(%clk: !pyc.clock, %rst: !pyc.reset, %in_valid: i1, %in_data: i129, %out_ready: i1) -> (i1, i64, i1) attributes {arg_names = ["clk", "rst", "in_valid", "in_data", "out_ready"], result_names = ["out_valid", "out_data", "in_ready"]} { + %input_pop = pyc.wire : i1 + %output_ready = pyc.wire : i1 + %in_ready, %queued_valid, %packet = pyc.fifo %clk, %rst, %in_valid, %in_data, %input_pop {depth = 1} : i129 + %is_remainder = pyc.extract %packet {lsb = 128} : i129 -> i1 + %lhs = pyc.extract %packet {lsb = 64} : i129 -> i64 + %rhs = pyc.extract %packet {lsb = 0} : i129 -> i64 + %signed = pyc.constant true : i1 + %word = pyc.constant false : i1 + %quotient, %remainder = pyc.divrem %lhs, %rhs, %signed, %word : (i64, i64, i1, i1) -> (i64, i64) + %selected = pyc.select %is_remainder, %remainder, %quotient : i1, i64, i64 -> i64 + %result_ready, %result_valid, %result = pyc.fifo %clk, %rst, %queued_valid, %selected, %output_ready {depth = 1} : i64 + pyc.assign %input_pop, %result_ready : i1 + pyc.assign %output_ready, %out_ready : i1 + func.return %result_valid, %result, %in_ready : i1, i64, i1 + } +} +""" + PYC_SEMANTIC_PRIMITIVES = { "priority_encode": """ func.func @priority(%value: i8) -> (i3, i1) attributes {result_names = ["index", "valid"]} { @@ -119,6 +140,19 @@ def test_comparison_annotation_describes_operands_and_result_is_i1(self) -> None self.assertNotIn("wire [7:0] same;", verilog) self.assertIn("assign same = left == right;", verilog) + def test_divrem_predecode_uses_retained_packet_until_response(self) -> None: + tool = load_tool() + verilog = tool.emit_verilog( + tool.parse_pyc_module(PYC_DIVREM_PACKET), + ROOT / "library/verilog", + ) + self.assertIn("assign is_remainder = divrem_held_packet[128];", verilog) + self.assertIn("assign lhs = divrem_held_packet[127:64];", verilog) + self.assertIn("assign rhs = divrem_held_packet[63:0];", verilog) + self.assertNotIn("assign is_remainder = packet[128];", verilog) + self.assertEqual(1, verilog.count("\n bsg_idiv_iterative #(")) + self.assertNotIn("pyc_runtime_div_comb", verilog) + def test_cli_rejects_invalid_timeout_and_path_aliases(self) -> None: tool = load_tool() with tempfile.TemporaryDirectory() as directory: diff --git a/tests/system/test_primitive_selection.py b/tests/system/test_primitive_selection.py index c95bb9139..0c19ef81f 100644 --- a/tests/system/test_primitive_selection.py +++ b/tests/system/test_primitive_selection.py @@ -16,6 +16,11 @@ def _root() -> Path: return Path(__file__).resolve().parents[2] +def _repository_text_digest(path: Path) -> str: + content = path.read_bytes().replace(b"\r\n", b"\n") + return "sha256:" + hashlib.sha256(content).hexdigest() + + def _tool(name: str) -> str: configured = os.environ.get(name.upper().replace("-", "_")) if configured and Path(configured).is_file(): @@ -45,8 +50,7 @@ def _primitive_width_module(widths: range) -> str: for width in widths: priority_width = max(1, (width - 1).bit_length()) count_width = max(1, width.bit_length()) - functions.append( - f""" func.func @width_{width}(%value: i{width}) + functions.append(f""" func.func @width_{width}(%value: i{width}) -> (i{priority_width}, i1, i{count_width}, i{count_width}) {{ %index, %valid = pyc.priority_encode %value {{order = \"low\"}} : i{width} -> i{priority_width}, i1 loc(\"width_{width}\":1:1) @@ -56,8 +60,7 @@ def _primitive_width_module(widths: range) -> str: i{width} -> i{count_width} loc(\"width_{width}\":3:1) func.return %index, %valid, %population, %zeros : i{priority_width}, i1, i{count_width}, i{count_width} - }}""" - ) + }}""") return "module {\n" + "\n".join(functions) + "\n}\n" @@ -155,7 +158,9 @@ def test_selector_is_catalog_owned_and_fail_closed(tmp_path: Path) -> None: catalog_document = json.loads(catalog.read_text(encoding="utf-8")) for implementation in catalog_document["implementations"]: for source in implementation["sources"]: - shutil.copy2(root / "library" / "verilog" / source["path"], isolated) + destination = isolated / source["path"] + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(root / "library" / "verilog" / source["path"], destination) (isolated / "licenses").mkdir() shutil.copy2( root / "library" / "verilog" / "licenses" / "BSD-3-Clause.txt", @@ -334,9 +339,9 @@ def test_popcount_selector_and_semantic_verifier(tmp_path: Path) -> None: def test_popcount_candidate_is_a_balanced_tree() -> None: - source = (_root() / "library/verilog/pyc_popcount_primitive.v").read_text( - encoding="utf-8" - ) + source = ( + _root() / "library/verilog/bitfield_primitives/pyc_popcount_primitive.sv" + ).read_text(encoding="utf-8") assert "always @*" not in source assert "count = count +" not in source assert "PAD_WIDTH = 1 << TREE_LEVELS" in source @@ -524,14 +529,14 @@ def test_popcount_pyc_cpp_and_selected_rtl_agree(tmp_path: Path) -> None: selection = manifest["rtl_selection"] assert selection["implementations"][0]["semantic_id"] == "pyc.popcount.v1" assert selection["implementations"][0]["sources"][0]["path"] == ( - "pyc_popcount_primitive.v" + "bitfield_primitives/pyc_popcount_primitive.sv" ) assert selection["bindings"][0]["parameters"] == { "COUNT_WIDTH": 4, "WIDTH": 13, } - cxx = shutil.which("c++") + cxx = os.environ.get("CXX") or shutil.which("c++") if cxx: harness = tmp_path / "popcount_harness.cpp" harness.write_text( @@ -574,7 +579,7 @@ def test_popcount_pyc_cpp_and_selected_rtl_agree(tmp_path: Path) -> None: ) assert executed.returncode == 0, executed.stderr - verilator = shutil.which("verilator") + verilator = os.environ.get("VERILATOR") or shutil.which("verilator") if verilator: linted = subprocess.run( [ @@ -677,7 +682,7 @@ def test_zero_count_pyc_cpp_and_selected_rtl_agree( for binding in selection["bindings"] ) - cxx = shutil.which("c++") + cxx = os.environ.get("CXX") or shutil.which("c++") if cxx: harness = tmp_path / "count_zeros_harness.cpp" harness.write_text( @@ -806,7 +811,7 @@ def test_popcount_rtl_candidate_handles_edge_widths(tmp_path: Path) -> None: "popcount_widths_tb", "-o", str(executable), - str(root / "library/verilog/pyc_popcount_primitive.v"), + str(root / "library/verilog/bitfield_primitives/pyc_popcount_primitive.sv"), str(testbench), ], text=True, @@ -916,7 +921,7 @@ def test_agentic_semantic_primitive_verilog_output_is_closed_and_lints( primitive_module: str, top: str, ) -> None: - verilator = shutil.which("verilator") + verilator = os.environ.get("VERILATOR") or shutil.which("verilator") if not verilator: pytest.skip("Verilator is unavailable") root = _root() @@ -1021,9 +1026,7 @@ def test_pyc_cpp_and_selected_rtl_agree(tmp_path: Path) -> None: source = implementations[0]["sources"][0] bundled_source = verilog / source["bundle_path"] assert bundled_source.is_file() - assert source["sha256"] == ( - "sha256:" + hashlib.sha256(bundled_source.read_bytes()).hexdigest() - ) + assert source["sha256"] == _repository_text_digest(bundled_source) bindings = selection["bindings"] assert len(bindings) == 2 assert {item["parameters"]["ORDER_LOW"] for item in bindings} == {0, 1} @@ -1031,7 +1034,7 @@ def test_pyc_cpp_and_selected_rtl_agree(tmp_path: Path) -> None: assert primitives.count("module pyc_priority_encode") == 1 assert "basejump" not in primitives.lower() - cxx = shutil.which("c++") + cxx = os.environ.get("CXX") or shutil.which("c++") if cxx: harness = tmp_path / "cpp_harness.cpp" harness.write_text( @@ -1066,7 +1069,7 @@ def test_pyc_cpp_and_selected_rtl_agree(tmp_path: Path) -> None: ) subprocess.run([str(executable)], cwd=root, check=True) - verilator = shutil.which("verilator") + verilator = os.environ.get("VERILATOR") or shutil.which("verilator") if verilator: subprocess.run( [ @@ -1122,5 +1125,5 @@ def test_installed_catalog_keeps_license_evidence() -> None: for implementation in catalog["implementations"]: license_path = catalog_path.parent / implementation["license_file"] assert license_path.is_file() - digest = "sha256:" + hashlib.sha256(license_path.read_bytes()).hexdigest() + digest = _repository_text_digest(license_path) assert digest == implementation["license_sha256"] diff --git a/tests/unit/test_primitive_catalog.py b/tests/unit/test_primitive_catalog.py index a7342d542..0d4eb0141 100644 --- a/tests/unit/test_primitive_catalog.py +++ b/tests/unit/test_primitive_catalog.py @@ -22,9 +22,16 @@ def test_implementation_catalog_is_bsd_and_digest_closed() -> None: assert catalog["schema"] == "pyc-rtl-catalog-v1" implementations = catalog["implementations"] assert {item["semantic_id"] for item in implementations} == { + "pyc.bitfield_clear.v1", + "pyc.bitfield_insert.v1", + "pyc.bitfield_set.v1", "pyc.count_zeros.v1", + "pyc.dynamic_sign_extend.v1", "pyc.popcount.v1", "pyc.priority_encode.v1", + "pyc.reverse_bytes.v1", + "pyc.runtime_zero_count.v1", + "pyc.wrapping_field_normalize.v1", } for implementation in implementations: assert implementation["effect_class"] == "comb" @@ -32,14 +39,16 @@ def test_implementation_catalog_is_bsd_and_digest_closed() -> None: assert implementation["license_file"] == "licenses/BSD-3-Clause.txt" license_path = catalog_path.parent / implementation["license_file"] assert license_path.is_file() + license_bytes = license_path.read_bytes().replace(b"\r\n", b"\n") assert implementation["license_sha256"] == ( - "sha256:" + hashlib.sha256(license_path.read_bytes()).hexdigest() + "sha256:" + hashlib.sha256(license_bytes).hexdigest() ) assert "basejump" not in implementation["implementation_id"].lower() for source in implementation["sources"]: assert source["license"] == "BSD-3-Clause" path = catalog_path.parent / source["path"] - digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + source_bytes = path.read_bytes().replace(b"\r\n", b"\n") + digest = "sha256:" + hashlib.sha256(source_bytes).hexdigest() assert source["sha256"] == digest @@ -77,6 +86,21 @@ def test_semantic_registry_contains_no_implementation_names() -> None: ] +def test_popcount_catalog_uses_the_single_bitfield_primitive_definition() -> None: + root = Path(__file__).resolve().parents[2] + catalog = json.loads( + (root / "library/verilog/rtl_catalog.json").read_text(encoding="utf-8") + ) + popcount = next( + item + for item in catalog["implementations"] + if item["semantic_id"] == "pyc.popcount.v1" + ) + assert [source["path"] for source in popcount["sources"]] == [ + "bitfield_primitives/pyc_popcount_primitive.sv" + ] + + def test_semantic_registry_width_formulas_cover_exact_shared_range() -> None: import pycircuit from _pycircuit_semantics import (