From fb38cd70a0e11bb1099085da16ad42cd2eb7a699 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:46:39 +0800 Subject: [PATCH 01/60] Forward GEP stride config packets --- controller/ControllerRTL.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controller/ControllerRTL.py b/controller/ControllerRTL.py index 83b41068..657ea56a 100644 --- a/controller/ControllerRTL.py +++ b/controller/ControllerRTL.py @@ -338,7 +338,8 @@ def update_received_msg(): (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_LAUNCH) | \ (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_LOWER) | \ (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \ - (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) : + (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ + (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) : s.recv_from_inter_cgra_noc.rdy @= s.send_to_ctrl_ring_pkt.rdy s.send_to_ctrl_ring_pkt.val @= s.recv_from_inter_cgra_noc.val s.send_to_ctrl_ring_pkt.msg @= \ From 00b4b026653168e7fb72dbdb8e133f2d51320fb9 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:46:42 +0800 Subject: [PATCH 02/60] Allow zero identity add predicate --- fu/single/AdderRTL.py | 15 ++++++++++++--- fu/single/test/AdderRTL_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/fu/single/AdderRTL.py b/fu/single/AdderRTL.py index 1a5b6833..58dcb180 100644 --- a/fu/single/AdderRTL.py +++ b/fu/single/AdderRTL.py @@ -67,9 +67,18 @@ def comb_logic(): if s.recv_opt.val: if s.recv_opt.msg.operation == OPT_ADD: s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload + s.recv_in[s.in1_idx].msg.payload - s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \ - s.recv_in[s.in1_idx].msg.predicate & \ - s.reached_vector_factor + if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate: + s.send_out[0].msg.predicate @= s.reached_vector_factor + elif s.recv_in[s.in0_idx].msg.predicate & \ + ~s.recv_in[s.in1_idx].msg.predicate & \ + (s.recv_in[s.in1_idx].msg.payload == s.const_zero.payload): + s.send_out[0].msg.predicate @= s.reached_vector_factor + elif s.recv_in[s.in1_idx].msg.predicate & \ + ~s.recv_in[s.in0_idx].msg.predicate & \ + (s.recv_in[s.in0_idx].msg.payload == s.const_zero.payload): + s.send_out[0].msg.predicate @= s.reached_vector_factor + else: + s.send_out[0].msg.predicate @= 0 s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val s.send_out[0].val @= s.recv_all_val s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy diff --git a/fu/single/test/AdderRTL_test.py b/fu/single/test/AdderRTL_test.py index 9608ea8d..973a8424 100644 --- a/fu/single/test/AdderRTL_test.py +++ b/fu/single/test/AdderRTL_test.py @@ -100,3 +100,31 @@ def test_alu(): src_const, src_opt, sink_out) run_sim(th) +def test_add_false_predicate_zero_is_identity(): + FU = AdderRTL + DataType = mk_data(16, 1) + num_inports = 2 + num_outports = 1 + ConfigType = mk_ctrl(num_inports, num_outports) + FuInType = mk_bits(clog2(num_inports + 1)) + + data_mem_size = 8 + ctrl_mem_size = 8 + DataAddrType = mk_bits(clog2(data_mem_size)) + CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) + CgraPayloadType = mk_cgra_payload(DataType, DataAddrType, ConfigType, CtrlAddrType) + IntraCgraPktType = mk_intra_cgra_pkt(1, 1, 1, CgraPayloadType) + + pickRegister = [FuInType(x + 1) for x in range(num_inports)] + src_in0 = [DataType(5, 1), DataType(5, 1), DataType(0, 0)] + src_in1 = [DataType(0, 0), DataType(2, 0), DataType(7, 1)] + src_const = [DataType(0, 0)] + sink_out = [DataType(5, 1), DataType(7, 0), DataType(7, 1)] + src_opt = [ConfigType(OPT_ADD, pickRegister), + ConfigType(OPT_ADD, pickRegister), + ConfigType(OPT_ADD, pickRegister)] + + th = TestHarness(FU, IntraCgraPktType, DataType, ConfigType, num_inports, + num_outports, data_mem_size, src_in0, src_in1, + src_const, src_opt, sink_out) + run_sim(th) From ce8b992613263e10d23157fa9bc1a60d96e39de6 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:45:53 +0800 Subject: [PATCH 03/60] Support const grant-once operations --- fu/single/GrantRTL.py | 46 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/fu/single/GrantRTL.py b/fu/single/GrantRTL.py index a5976719..7001e32b 100644 --- a/fu/single/GrantRTL.py +++ b/fu/single/GrantRTL.py @@ -30,7 +30,11 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0 s.in0_idx = Wire(idx_nbits) s.in1_idx = Wire(idx_nbits) s.recv_all_val = Wire(1) - s.already_grt_once = Wire(1) + # Per-slot (per ctrl_addr) latch so that multiple GRANT_ONCE ops in the + # same tile do not share state. + num_slots = 1 << s.CtrlAddrType.nbits + s.already_grt_once = [Wire(1) for _ in range(num_slots)] + s.cur_already_grt_once = Wire(1) # Connections. s.in0_idx //= s.in0[0:idx_nbits] @@ -56,6 +60,9 @@ def comb_logic(): s.send_to_ctrl_mem.msg @= s.CgraPayloadType(0, 0, 0, 0, 0) s.recv_from_ctrl_mem.rdy @= 0 + # Select the per-slot "already granted" bit for the current ctrl_addr. + s.cur_already_grt_once @= s.already_grt_once[s.ctrl_addr_inport] + if s.recv_opt.val: if s.recv_opt.msg.fu_in[0] != FuInType(0): s.in0 @= s.recv_opt.msg.fu_in[0] - FuInType(1) @@ -100,13 +107,25 @@ def comb_logic(): # GRANT_ONCE is used to apply `true` predicate onto a value only once. This # is usually used for the constant declared in the entry block of a function. s.send_out[0].msg @= s.recv_in[s.in0_idx].msg - # Only updates predicate as true for the first time. - s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.already_grt_once + # Only updates predicate as true for the first time at this ctrl_addr. + s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.cur_already_grt_once s.recv_all_val @= s.recv_in[s.in0_idx].val s.send_out[0].val @= s.recv_all_val s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy + elif s.recv_opt.msg.operation == OPT_GRT_ONCE_CONST: + # GRANT_ONCE_CONST: every non-prologue execution consumes one entry from + # the const queue, but predicate=1 is emitted only on the first such + # execution for this ctrl_addr; subsequent executions emit predicate=0. + # (Prologue cycles see OPT_NAH via FlexibleFuRTL and do not reach here.) + s.send_out[0].msg @= s.recv_const.msg + s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.cur_already_grt_once + + s.recv_all_val @= s.recv_const.val + s.send_out[0].val @= s.recv_all_val + s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy + s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy else: for j in range( num_outports ): @@ -117,10 +136,19 @@ def comb_logic(): @update_ff def record_grt_once(): - if s.reset | s.clear: - s.already_grt_once <<= 0 - else: - if ~s.already_grt_once & s.send_out[0].val & s.send_out[0].rdy & (s.recv_opt.msg.operation == OPT_GRT_ONCE): - s.already_grt_once <<= 1 + for k in range(num_slots): + if s.reset | s.clear: + s.already_grt_once[k] <<= 0 else: - s.already_grt_once <<= s.already_grt_once + # Latch only the slot that just fired a successful GRANT_ONCE / + # GRANT_ONCE_CONST; all other slots hold their state. This gives + # each ctrl_addr an independent "already granted" flag so multiple + # GRANT_ONCE ops in the same tile do not share state. + if (s.ctrl_addr_inport == s.CtrlAddrType(k)) & \ + ~s.already_grt_once[k] & \ + s.send_out[0].val & s.send_out[0].rdy & \ + ((s.recv_opt.msg.operation == OPT_GRT_ONCE) | \ + (s.recv_opt.msg.operation == OPT_GRT_ONCE_CONST)): + s.already_grt_once[k] <<= 1 + else: + s.already_grt_once[k] <<= s.already_grt_once[k] From ab5d3412c34ba50dc12dd218ab1cd2ba60911402 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:45:53 +0800 Subject: [PATCH 04/60] Add small conv YAML validation --- cgra/test/CgraRTL_conv4x4_test_from_yaml.py | 1052 +++++++++++++++++++ validation/script_generator.py | 961 ++++++++++++----- validation/test/conv/conv_small.yaml | 474 +++++++++ 3 files changed, 2227 insertions(+), 260 deletions(-) create mode 100644 cgra/test/CgraRTL_conv4x4_test_from_yaml.py create mode 100644 validation/test/conv/conv_small.yaml diff --git a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py new file mode 100644 index 00000000..2c594024 --- /dev/null +++ b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py @@ -0,0 +1,1052 @@ +""" +========================================================================== +CgraRTL_conv4x4_test_from_yaml.py +========================================================================== +Test cases for CGRA with crossbar-based data memory and ring-based control +memory of each tile, using the generated conv kernel. + +Conv kernel semantics (SMALL_DATASET, NI=60, NJ=70, total=4200): + out = 0 + for x = 0 to 4199: (ICMP_EQ #4200) + i = x / 70 (DIV #70) + j = x % 70 (REM #70) + out += A[i][j] * B[i][j] (GEP + LOAD + MUL + ADD) + return out (RETURN_VALUE) + +A is stored at addresses [0..4199], B at addresses [4200..8399]. +A[i][j] = A[i*70+j], B[i][j] = B[i*70+j]. + +Author : Shiran Guo + Date : Apr 6, 2026 +""" + +import os +import time +import yaml + +from pymtl3.datatypes import b1, b2 +from pymtl3.passes.backends.verilog import (VerilogVerilatorImportPass) +from pymtl3.passes.backends.verilog.import_.VerilogVerilatorImportConfigs import ( + VerilogVerilatorImportConfigs, +) +from pymtl3.stdlib.test_utils import (run_sim, + config_model_with_cmdline_opts) + +from ..CgraRTL import CgraRTL +from ...fu.double.SeqMulAdderRTL import SeqMulAdderRTL +from ...fu.flexible.FlexibleFuRTL import FlexibleFuRTL +from ...fu.float.FpAddRTL import FpAddRTL +from ...fu.float.FpMulRTL import FpMulRTL +from ...fu.single.AdderRTL import AdderRTL +from ...fu.single.DivRTL import DivRTL +from ...fu.single.GepRTL import GepRTL +from ...fu.single.GrantRTL import GrantRTL +from ...fu.single.CompRTL import CompRTL +from ...fu.single.LogicRTL import LogicRTL +from ...fu.single.MemUnitRTL import MemUnitRTL +from ...fu.single.MulRTL import MulRTL +from ...fu.single.PhiRTL import PhiRTL +from ...fu.single.RetRTL import RetRTL +from ...fu.single.SelRTL import SelRTL +from ...fu.single.ShifterRTL import ShifterRTL +from ...fu.vector.VectorAdderComboRTL import VectorAdderComboRTL +from ...fu.vector.VectorMulComboRTL import VectorMulComboRTL +from ...fu.vector.VectorAllReduceRTL import VectorAllReduceRTL +from ...lib.basic.val_rdy.SinkRTL import SinkRTL as TestSinkRTL +from ...lib.basic.val_rdy.SourceRTL import SourceRTL as TestSrcRTL +from ...lib.messages import * +from ...lib.opt_type import * +from ...lib.util.common import * +try: + from ...lib.trace_logger import init_trace_logger, close_trace_logger +except ModuleNotFoundError: + def init_trace_logger(*args, **kwargs): + return None + + def close_trace_logger(): + return None + +def patch_conv_verilator_import(): + """Keep the large conv Verilator import compile from forming one huge TU.""" + + if getattr(VerilogVerilatorImportConfigs, + "_conv4x4_light_import_patched", False): + return + + def create_vl_cmd(s): + top_module = f"--top-module {s.translated_top_module}" + src = s.translated_source_file + mk_dir = f"--Mdir {s.vl_mk_dir}" + include = "" if not s.v_include else \ + " ".join("-I" + path for path in s.v_include) + en_assert = "--assert" if s.vl_enable_assert else "" + opt_level = os.environ.get("CGRA_VERILATOR_OPT_LEVEL", "3") + opt = f"-O{opt_level}" + loop_unroll = "--unroll-count {}".format( + os.environ.get("CGRA_VERILATOR_UNROLL_COUNT", "1000000") + ) + stmt_unroll = "--unroll-stmts {}".format( + os.environ.get("CGRA_VERILATOR_UNROLL_STMTS", "1000000") + ) + output_split = os.environ.get("CGRA_VERILATOR_OUTPUT_SPLIT", "20000") + split = "" + if int(output_split) > 0: + split = "--output-split {0} --output-split-cfuncs {0}".format( + output_split + ) + trace = "--trace" if s.vl_trace else "" + coverage = "--coverage" if s.vl_coverage else "" + line_cov = "--coverage-line" if s.vl_line_coverage else "" + toggle_cov = "--coverage-toggle" if s.vl_toggle_coverage else "" + warnings = s._create_vl_warning_cmd() + vlibs = "" + + all_opts = [ + top_module, mk_dir, include, en_assert, opt, loop_unroll, stmt_unroll, + split, trace, warnings, src, vlibs, coverage, line_cov, toggle_cov, + ] + return "verilator --cc {}".format( + " ".join(opt for opt in all_opts if opt) + ) + + def get_c_src_files_split(s): + top_module = s.translated_top_module + vl_mk_dir = s.vl_mk_dir + vl_class_mk = f"{vl_mk_dir}/V{top_module}_classes.mk" + + with open(vl_class_mk) as class_mk: + all_lines = class_mk.readlines() + + fast_srcs = list(s.c_srcs) + [s.get_c_wrapper_path()] + fast_srcs += s._get_srcs_from_vl_class_mk( + all_lines, vl_mk_dir, "VM_CLASSES_FAST" + ) + fast_srcs += s._get_srcs_from_vl_class_mk( + all_lines, vl_mk_dir, "VM_SUPPORT_FAST" + ) + fast_srcs += s._get_srcs_from_vl_class_mk( + all_lines, s.vl_include_dir, "VM_GLOBAL_FAST" + ) + + slow_srcs = [] + slow_srcs += s._get_srcs_from_vl_class_mk( + all_lines, vl_mk_dir, "VM_CLASSES_SLOW" + ) + slow_srcs += s._get_srcs_from_vl_class_mk( + all_lines, vl_mk_dir, "VM_SUPPORT_SLOW" + ) + slow_srcs += s._get_srcs_from_vl_class_mk( + all_lines, s.vl_include_dir, "VM_GLOBAL_SLOW" + ) + return fast_srcs + slow_srcs + + VerilogVerilatorImportConfigs.create_vl_cmd = create_vl_cmd + if os.environ.get("CGRA_VERILATOR_SEPARATE_COMPILE", "1") != "0": + VerilogVerilatorImportConfigs._get_c_src_files = get_c_src_files_split + VerilogVerilatorImportConfigs._conv4x4_light_import_patched = True + +#------------------------------------------------------------------------- +# Test harness +#------------------------------------------------------------------------- + +class TestHarness(Component): + + def construct(s, DUT, FunctionUnit, FuList, + CtrlPktType, + cgra_id, width, height, + ctrl_mem_size, data_mem_size_global, + data_mem_size_per_bank, num_banks_per_cgra, + num_registers_per_reg_bank, + src_ctrl_pkt, kCtrlCountPerIter, kTotalCtrlSteps, + mem_access_is_combinational, controller2addr_map, + idTo2d_map, complete_signal_sink_out, + multi_cgra_rows, multi_cgra_columns, src_query_pkt, + preload_pkt_count = 0, preload_drain_cycles = 0): + + CgraPayloadType = CtrlPktType.get_field_type(kAttrPayload) + DataType = CgraPayloadType.get_field_type(kAttrData) + DataAddrType = mk_bits(clog2(data_mem_size_global)) + s.num_tiles = width * height + s.src_ctrl_pkt = TestSrcRTL(CtrlPktType, src_ctrl_pkt) + s.src_query_pkt = TestSrcRTL(CtrlPktType, src_query_pkt) + + s.dut = DUT(CgraPayloadType, + # CGRA terminals on x/y. Assume in total 4, though this + # test is for single CGRA. + multi_cgra_rows, multi_cgra_columns, + width, height, ctrl_mem_size, + data_mem_size_global, data_mem_size_per_bank, + num_banks_per_cgra, num_registers_per_reg_bank, + kCtrlCountPerIter, kTotalCtrlSteps, + mem_access_is_combinational, + FunctionUnit, FuList, "Mesh", + controller2addr_map, idTo2d_map, + is_multi_cgra = False) + + cmp_fn = lambda a, b: \ + (a.payload.cmd == b.payload.cmd) and \ + (a.payload.data.payload == b.payload.data.payload) and \ + (a.payload.data.predicate == b.payload.data.predicate) + s.complete_signal_sink_out = TestSinkRTL(CtrlPktType, complete_signal_sink_out, cmp_fn = cmp_fn) + + # Connections + s.dut.cgra_id //= cgra_id + + expected_return_src = int(complete_signal_sink_out[0].src) \ + if complete_signal_sink_out else -1 + expected_return_data = int(complete_signal_sink_out[0].payload.data.payload) \ + if complete_signal_sink_out else 0 + expected_complete_pkt = complete_signal_sink_out[0] \ + if complete_signal_sink_out else CtrlPktType() + + complete_count_value = \ + sum(1 for pkt in complete_signal_sink_out \ + if pkt.payload.cmd == CMD_COMPLETE) + + CompleteCountType = mk_bits(clog2(complete_count_value + 1)) + s.complete_count = Wire(CompleteCountType) + StoreCountType = mk_bits(clog2(max(preload_pkt_count, 1) + 1)) + DrainCountType = mk_bits(clog2(max(preload_drain_cycles, 1) + 1)) + s.preload_sent_count = Wire(StoreCountType) + s.preload_drain_count = Wire(DrainCountType) + s.preload_draining = Wire(1) + + @update + def update_preload_draining(): + if preload_pkt_count > 0: + s.preload_draining @= \ + (s.preload_sent_count >= StoreCountType(preload_pkt_count)) & \ + (s.preload_drain_count < DrainCountType(preload_drain_cycles)) + else: + s.preload_draining @= 0 + + @update + def conditional_issue_ctrl_or_query(): + s.dut.recv_from_cpu_pkt.val @= s.src_ctrl_pkt.send.val + s.dut.recv_from_cpu_pkt.msg @= s.src_ctrl_pkt.send.msg + s.src_ctrl_pkt.send.rdy @= 0 + s.src_query_pkt.send.rdy @= 0 + if (s.complete_count >= complete_count_value) & \ + ~s.src_ctrl_pkt.send.val: + s.dut.recv_from_cpu_pkt.val @= s.src_query_pkt.send.val + s.dut.recv_from_cpu_pkt.msg @= s.src_query_pkt.send.msg + s.src_query_pkt.send.rdy @= s.dut.recv_from_cpu_pkt.rdy + else: + s.src_ctrl_pkt.send.rdy @= s.dut.recv_from_cpu_pkt.rdy + + if s.preload_draining: + s.dut.recv_from_cpu_pkt.val @= 0 + s.src_ctrl_pkt.send.rdy @= 0 + + skip_bad_returns = os.environ.get("CGRA_SKIP_BAD_RETURNS", "0") == "1" + + @update + def filter_return_complete(): + s.complete_signal_sink_out.recv.val @= 0 + s.complete_signal_sink_out.recv.msg @= expected_complete_pkt + s.dut.send_to_cpu_pkt.rdy @= 1 + + if s.dut.send_to_cpu_pkt.val & \ + (s.dut.send_to_cpu_pkt.msg.src == expected_return_src) & \ + (s.dut.send_to_cpu_pkt.msg.payload.cmd == CMD_COMPLETE): + if skip_bad_returns & \ + (s.dut.send_to_cpu_pkt.msg.payload.data.payload != + expected_return_data): + s.complete_signal_sink_out.recv.val @= 0 + s.dut.send_to_cpu_pkt.rdy @= 1 + else: + s.complete_signal_sink_out.recv.val @= 1 + s.dut.send_to_cpu_pkt.rdy @= s.complete_signal_sink_out.recv.rdy + + @update_ff + def update_complete_count(): + if s.reset: + s.complete_count <<= 0 + else: + if s.complete_signal_sink_out.recv.val & s.complete_signal_sink_out.recv.rdy & \ + (s.complete_count < complete_count_value): + s.complete_count <<= s.complete_count + CompleteCountType(1) + + @update_ff + def update_preload_counts(): + if s.reset: + s.preload_sent_count <<= 0 + s.preload_drain_count <<= 0 + else: + if preload_pkt_count > 0: + if s.dut.recv_from_cpu_pkt.val & s.dut.recv_from_cpu_pkt.rdy & \ + (s.dut.recv_from_cpu_pkt.msg.payload.cmd == CMD_STORE_REQUEST) & \ + (s.preload_sent_count < StoreCountType(preload_pkt_count)): + s.preload_sent_count <<= s.preload_sent_count + StoreCountType(1) + if (s.preload_sent_count >= StoreCountType(preload_pkt_count)) & \ + (s.preload_drain_count < DrainCountType(preload_drain_cycles)): + s.preload_drain_count <<= s.preload_drain_count + DrainCountType(1) + + # Connects memory address upper and lower bound for each CGRA. + s.dut.address_lower //= DataAddrType(controller2addr_map[cgra_id][0]) + s.dut.address_upper //= DataAddrType(controller2addr_map[cgra_id][1]) + + for tile_col in range(width): + s.dut.send_data_on_boundary_north[tile_col].rdy //= 0 + s.dut.recv_data_on_boundary_north[tile_col].val //= 0 + s.dut.recv_data_on_boundary_north[tile_col].msg //= DataType() + + s.dut.send_data_on_boundary_south[tile_col].rdy //= 0 + s.dut.recv_data_on_boundary_south[tile_col].val //= 0 + s.dut.recv_data_on_boundary_south[tile_col].msg //= DataType() + + for tile_row in range(height): + s.dut.send_data_on_boundary_west[tile_row].rdy //= 0 + s.dut.recv_data_on_boundary_west[tile_row].val //= 0 + s.dut.recv_data_on_boundary_west[tile_row].msg //= DataType() + + s.dut.send_data_on_boundary_east[tile_row].rdy //= 0 + s.dut.recv_data_on_boundary_east[tile_row].val //= 0 + s.dut.recv_data_on_boundary_east[tile_row].msg //= DataType() + + def done(s): + return (s.src_ctrl_pkt.done() and s.src_query_pkt.done() + and s.complete_signal_sink_out.done()) + + def line_trace(s): + return s.dut.line_trace() + +# Common configurations/setups. +FuList = [AdderRTL, + MulRTL, + DivRTL, + GepRTL, + LogicRTL, + ShifterRTL, + PhiRTL, + CompRTL, + GrantRTL, + MemUnitRTL, + SelRTL, + RetRTL, + ] +x_tiles = 4 +y_tiles = 4 +data_bitwidth = 32 +tile_ports = 4 +num_tile_inports = tile_ports +num_tile_outports = tile_ports +num_fu_inports = 4 +num_fu_outports = 2 +num_routing_outports = num_tile_outports + num_fu_inports +ctrl_mem_size = 6 +data_mem_size_global = 65536 +data_mem_size_per_bank = 8192 +num_banks_per_cgra = 2 +num_cgra_columns = 4 +num_cgra_rows = 1 +num_cgras = num_cgra_columns * num_cgra_rows +num_ctrl_operations = 64 +num_registers_per_reg_bank = 8 +TileInType = mk_bits(clog2(num_tile_inports + num_fu_inports + 1)) +FuInType = mk_bits(clog2(num_fu_inports + 1)) +FuOutType = mk_bits(clog2(num_fu_outports + 1)) +addr_nbits = clog2(data_mem_size_global) +num_tiles = x_tiles * y_tiles +num_rd_tiles = x_tiles + y_tiles - 1 +per_cgra_data_size = int(data_mem_size_global / num_cgras) + +DUT = CgraRTL +FunctionUnit = FlexibleFuRTL + +DataAddrType = mk_bits(addr_nbits) +RegIdxType = mk_bits(clog2(num_registers_per_reg_bank)) +DataType = mk_data(data_bitwidth, 1) +PredicateType = mk_predicate(1, 1) +ControllerIdType = mk_bits(max(1, clog2(num_cgras))) +cgra_id = 0 +controller2addr_map = {} +for i in range(num_cgras): + controller2addr_map[i] = [i * per_cgra_data_size, + (i + 1) * per_cgra_data_size - 1] +idTo2d_map = { + 0: [0, 0], + 1: [1, 0], + 2: [2, 0], + 3: [3, 0], +} + +cgra_id_nbits = clog2(num_cgras) +addr_nbits = clog2(data_mem_size_global) +predicate_nbits = 1 + +CtrlType = mk_ctrl(num_fu_inports, + num_fu_outports, + num_tile_inports, + num_tile_outports, + num_registers_per_reg_bank) + +CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) + +CgraPayloadType = mk_cgra_payload(DataType, + DataAddrType, + CtrlType, + CtrlAddrType) + +InterCgraPktType = mk_inter_cgra_pkt(num_cgra_columns, + num_cgra_rows, + num_tiles, + num_rd_tiles, + CgraPayloadType) + +IntraCgraPktType = mk_intra_cgra_pkt(num_cgra_columns, + num_cgra_rows, + num_tiles, + CgraPayloadType) + +# Helper to convert signed int to unsigned 32-bit representation. +def to_uint32(val): + """Convert signed Python int to 32-bit unsigned representation.""" + if val < 0: + return val + (1 << data_bitwidth) + return val + + +# ======================================================================== +# Conv kernel parameters (generated SMALL_DATASET: NI=60, NJ=70, total=4200) +# ======================================================================== +# Memory layout: +# A[0..4199] at addresses 0..4199 (base_A = 0) +# B[0..4199] at addresses 4200..8399 (base_B = 4200) +# +# Use explicit non-zero data so the numerical check is meaningful. The env +# overrides are only for reduced debug runs with a matching generated YAML. + +NI = int(os.environ.get("CGRA_CONV_NI", "60")) +NJ = int(os.environ.get("CGRA_CONV_NJ", "70")) +total = NI * NJ # 4200 +base_A = 0 +base_B = total + +A_values = [1 for _ in range(total)] +B_values = [1 for _ in range(total)] + +expected_result = sum(a * b for a, b in zip(A_values, B_values)) # 4200 +conv_max_scheduled_time_step = 12 +conv_ctrl_count_per_iter = 5 + +def preload_word(dut, addr, value): + bank = addr // data_mem_size_per_bank + bank_addr = addr % data_mem_size_per_bank + dut.data_mem.memory_wrapper[bank].memory.regs[bank_addr] <<= \ + DataType(value, 1) + + +def preload_conv_data(dut): + for i in range(total): + preload_word(dut, base_A + i, A_values[i]) + preload_word(dut, base_B + i, B_values[i]) + if os.environ.get("CGRA_DEBUG_PRELOAD", "0") == "1": + probe_addrs = [base_A, base_A + total - 1, base_B, base_B + total - 1] + for addr in probe_addrs: + bank = addr // data_mem_size_per_bank + bank_addr = addr % data_mem_size_per_bank + word = dut.data_mem.memory_wrapper[bank].memory.regs[bank_addr] + print("[preload_probe]", + "addr", addr, + "bank", bank, + "bank_addr", bank_addr, + "payload", int(word.payload), + "predicate", int(word.predicate), + flush=True) + + +def make_preload_packets(): + return [ + IntraCgraPktType(0, 0, payload = CgraPayloadType(CMD_STORE_REQUEST, + data = DataType(A_values[i], 1), data_addr = base_A + i)) + for i in range(total) + ] + [ + IntraCgraPktType(0, 0, payload = CgraPayloadType(CMD_STORE_REQUEST, + data = DataType(B_values[i], 1), data_addr = base_B + i)) + for i in range(total) + ] + + +def make_preload_probe_packets(): + return [ + IntraCgraPktType(0, 0, payload = CgraPayloadType(CMD_LOAD_REQUEST, + data_addr = base_A)), + IntraCgraPktType(0, 0, payload = CgraPayloadType(CMD_LOAD_REQUEST, + data_addr = base_B)), + ] + + +def find_return_src_from_yaml(path): + with open(path, "r") as yaml_file: + yaml_struct = yaml.safe_load(yaml_file) + + for core in yaml_struct["array_config"]["cores"]: + for entry in core["entries"]: + for instruction in entry["instructions"]: + for operation in instruction["operations"]: + if operation["opcode"] in ("RETURN", "RETURN_VALUE", "RETURN_VOID"): + if "core_id" in core: + return int(core["core_id"]) + return int(core["row"]) * int(yaml_struct["array_config"]["columns"]) + \ + int(core["column"]) + + raise AssertionError(f"missing RETURN_VALUE operation in {path}") + + +def sim_conv(cmdline_opts, mem_access_is_combinational): + src_ctrl_pkt = [] + complete_signal_sink_out = [] + src_query_pkt = [] + + # Kernel specific parameters matching conv-instructions.yaml. + kLoopLowerBound = 0 # GRANT_ONCE #0 + kLoopIncrement = 1 # ADD #1 + kLoopUpperBound = total # ICMP_EQ #4200 + kCtrlCountPerIter = conv_ctrl_count_per_iter + # RETURN_VALUE is at time_step 12. The terminal predicate is produced by the + # final loop-control iteration and still needs the scheduled tail to traverse + # back to the return tile. + kMaxScheduledTimeStep = conv_max_scheduled_time_step + kTotalCtrlSteps = kCtrlCountPerIter * \ + (kLoopUpperBound - kLoopLowerBound) + \ + kMaxScheduledTimeStep - kCtrlCountPerIter + 1 + kDutTotalCtrlSteps = kTotalCtrlSteps + + from ...validation.script_generator import ScriptFactory + conv_yaml_path = os.environ.get("CGRA_CONV_YAML", + "validation/test/conv/tmp-generated-instructions.yaml") + expected_return_src = find_return_src_from_yaml(conv_yaml_path) + script_factory = ScriptFactory(path = conv_yaml_path, + CtrlType = CtrlType, + IntraCgraPktType = IntraCgraPktType, + CgraPayloadType = CgraPayloadType, + TileInType = TileInType, + FuOutType = FuOutType, + CMD_CONFIG_input = CMD_CONFIG, + FuInType=FuInType, + ii = kCtrlCountPerIter, + loop_times = kTotalCtrlSteps, + CMD_CONST_input = CMD_CONST, + CMD_CONFIG_COUNT_PER_ITER_input = CMD_CONFIG_COUNT_PER_ITER, + CMD_CONFIG_TOTAL_CTRL_COUNT_input = CMD_CONFIG_TOTAL_CTRL_COUNT, + CMD_CONFIG_PROLOGUE_FU_input = CMD_CONFIG_PROLOGUE_FU, + CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR_input = CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR, + CMD_CONFIG_PROLOGUE_FU_CROSSBAR_input = CMD_CONFIG_PROLOGUE_FU_CROSSBAR, + CMD_LAUNCH_input = CMD_LAUNCH, + DataType = DataType, + B1Type = b1, + B2Type = b2, + RegIdxType = RegIdxType, + CtrlAddrType = CtrlAddrType, + DataAddrType = DataAddrType, + num_registers_per_reg_bank = num_registers_per_reg_bank, + arg_map = { + "arg6": base_A, # base address of array A + "arg7": base_B, # base address of array B + }, + gep_stride = NJ, # stride for 2D GEP = NJ + accumulate_add_to_src_reg = True) + + src_opt_pkt0_ = script_factory.makeVectorCGRAPkts() + + # order the packets according to the x (first) and y (second) coordinates + src_opt_pkt0 = [] + for x, y in src_opt_pkt0_: + src_opt_pkt0.append(src_opt_pkt0_[(x, y)]) + + src_query_pkt = \ + [ + ] + + # RETURN_VALUE sends CMD_COMPLETE with data = expected_result. + expected_complete_sink_out_pkg = \ + [ + IntraCgraPktType(src = expected_return_src, dst = 16, + payload = CgraPayloadType(CMD_COMPLETE, + DataType(expected_result, 1, 0, 0))) + for _ in range(1) + ] + expected_mem_sink_out_pkt = \ + [ + ] + + print("src_opt_pkt0 tiles:", [len(tile_pkts) for tile_pkts in src_opt_pkt0], + flush=True) + + use_verilator = os.environ.get("CGRA_USE_VERILATOR", "0") == "1" + preload_pkt_count = 0 + preload_drain_cycles = 0 + + if use_verilator: + src_ctrl_pkt.extend(make_preload_packets()) + preload_pkt_count = 2 * total + preload_drain_cycles = int(os.environ.get("CGRA_PRELOAD_DRAIN_CYCLES", "10000")) + if os.environ.get("CGRA_PRELOAD_PROBE", "0") == "1": + src_ctrl_pkt.extend(make_preload_probe_packets()) + + for tile_pkts in src_opt_pkt0: + src_ctrl_pkt.extend(tile_pkts) + + complete_signal_sink_out.extend(expected_complete_sink_out_pkg) + complete_signal_sink_out.extend(expected_mem_sink_out_pkt) + + t0 = time.time() + print("[timing] construct harness", flush=True) + th = TestHarness(DUT, FunctionUnit, FuList, + IntraCgraPktType, + cgra_id, x_tiles, y_tiles, + ctrl_mem_size, data_mem_size_global, + data_mem_size_per_bank, num_banks_per_cgra, + num_registers_per_reg_bank, + src_ctrl_pkt, kCtrlCountPerIter, kDutTotalCtrlSteps, + mem_access_is_combinational, + controller2addr_map, idTo2d_map, complete_signal_sink_out, + num_cgra_rows, num_cgra_columns, + src_query_pkt, + preload_pkt_count = preload_pkt_count, + preload_drain_cycles = preload_drain_cycles) + + from pymtl3 import DefaultPassGroup + if use_verilator: + patch_conv_verilator_import() + print(f"[timing] harness constructed in {time.time() - t0:.2f}s", + flush=True) + t0 = time.time() + print("[timing] elaborate", flush=True) + th.elaborate() + print(f"[timing] elaborate done in {time.time() - t0:.2f}s", flush=True) + t0 = time.time() + print("[timing] Verilator translate/import", flush=True) + th.dut.set_metadata(VerilogVerilatorImportPass.vl_Wno_list, + ['UNSIGNED', 'UNOPTFLAT', 'WIDTH', 'WIDTHCONCAT', + 'ALWCOMBORDER']) + th.dut.set_metadata( + VerilogVerilatorImportPass.vl_mk_dir, + os.environ.get("CGRA_VERILATOR_MK_DIR", "obj_dir_conv4x4_light"), + ) + verilator_opts = dict(cmdline_opts) + verilator_opts["test_verilog"] = "zeros" + th = config_model_with_cmdline_opts(th, verilator_opts, duts = ['dut']) + print(f"[timing] Verilator import done in {time.time() - t0:.2f}s", + flush=True) + t0 = time.time() + print("[timing] apply DefaultPassGroup", flush=True) + th.apply(DefaultPassGroup(linetrace=False)) + print(f"[timing] apply done in {time.time() - t0:.2f}s", flush=True) + t0 = time.time() + print("[timing] reset", flush=True) + th.sim_reset() + print(f"[timing] reset done in {time.time() - t0:.2f}s", flush=True) + else: + # Use pure-Python simulation so we can inspect internal signals for + # debugging. Direct preload avoids spending thousands of cycles on + # store packets. + print(f"[timing] harness constructed in {time.time() - t0:.2f}s", + flush=True) + t0 = time.time() + print("[timing] elaborate", flush=True) + th.elaborate() + print(f"[timing] elaborate done in {time.time() - t0:.2f}s", flush=True) + t0 = time.time() + print("[timing] apply DefaultPassGroup", flush=True) + th.apply(DefaultPassGroup(linetrace=False)) + print(f"[timing] apply done in {time.time() - t0:.2f}s", flush=True) + t0 = time.time() + print("[timing] reset/preload", flush=True) + th.sim_reset() + preload_conv_data(th.dut) + print(f"[timing] reset/preload done in {time.time() - t0:.2f}s", + flush=True) + + trace_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'trace_output') + trace_file = os.path.join(trace_dir, 'trace_conv4x4_4x4_Mesh.jsonl') + trace_enabled = (os.environ.get("CGRA_TRACE_EVERY_CYCLE", "0") == "1") \ + and not use_verilator + progress_enabled = os.environ.get("CGRA_PROGRESS_LOG", "0") == "1" + debug_progress_enabled = os.environ.get("CGRA_DEBUG_PROGRESS", "0") == "1" + debug_every = int(os.environ.get("CGRA_DEBUG_EVERY", "1000")) + debug_from_cycle = int(os.environ.get("CGRA_DEBUG_FROM_CYCLE", "0")) + heartbeat_enabled = os.environ.get("CGRA_HEARTBEAT", "1") != "0" + debug_tile_ids = [2, 3, 5, 6, 7, 9, 10, 11] + debug_elem_tile_ids = [ + int(x) for x in os.environ.get("CGRA_DEBUG_ELEM_TILES", + "2,5,6,9,10").split(",") if x + ] + debug_route_tile_ids = [ + int(x) for x in os.environ.get("CGRA_DEBUG_ROUTE_TILES", + "2,5,6,9,10").split(",") if x + ] + trace_logger = init_trace_logger(trace_file, x_tiles, y_tiles, "Mesh", cgra_id) \ + if trace_enabled else None + + MAX_CYCLES = int(os.environ.get("CGRA_MAX_CYCLES", "60000")) + active_tiles = {} if use_verilator else { + 2: th.dut.tile[2], 3: th.dut.tile[3], + 5: th.dut.tile[5], 6: th.dut.tile[6], + 7: th.dut.tile[7], 9: th.dut.tile[9], + 10: th.dut.tile[10], 11: th.dut.tile[11], + } + + prev_state = {} + for tid in active_tiles: + prev_state[tid] = (-1, -1) # (raddr, rdcur) + stall_count = 0 + + for cycle in range(MAX_CYCLES): + th.sim_tick() + if trace_enabled: + trace_logger.log_cycle(th.dut) + if int(th.dut.send_to_cpu_pkt.val) & int(th.dut.send_to_cpu_pkt.rdy): + cpu_pkt = th.dut.send_to_cpu_pkt.msg + print("cpu_pkt:", + "cycle", cycle, + "src", int(cpu_pkt.src), + "cmd", int(cpu_pkt.payload.cmd), + "data", int(cpu_pkt.payload.data.payload), + "pred", int(cpu_pkt.payload.data.predicate), + "byp", int(cpu_pkt.payload.data.bypass), + "delay", int(cpu_pkt.payload.data.delay), + flush=True) + if int(cpu_pkt.payload.data.payload) != expected_result: + if hasattr(th.dut, "debug_tile_times"): + print("[mismatch_tile6]", + "times", int(th.dut.debug_tile_times[6]), + "addr", int(th.dut.debug_tile_ctrl_addr[6]), + "op", int(th.dut.debug_tile_op[6]), + "reg0_b0", int(th.dut.debug_tile_reg0_data[6][0]), + "reg0_b0p", int(th.dut.debug_tile_reg0_pred[6][0]), + "reg0_b1", int(th.dut.debug_tile_reg0_data[6][1]), + "reg0_b1p", int(th.dut.debug_tile_reg0_pred[6][1]), + "reg_rd_b0", int(th.dut.debug_tile_reg_read_data[6][0]), + "reg_rd_b0p", int(th.dut.debug_tile_reg_read_pred[6][0]), + "reg_rd_b1", int(th.dut.debug_tile_reg_read_data[6][1]), + "reg_rd_b1p", int(th.dut.debug_tile_reg_read_pred[6][1]), + flush=True) + else: + t6 = th.dut.tile[6] + cm = t6.ctrl_mem + print("[mismatch_tile6]", + "raddr", int(cm.reg_file.raddr[0]), + "times", int(cm.times), + "op", int(cm.send_ctrl.msg.operation), + "reg0_b0", int(t6.register_cluster.debug_reg0[0].payload), + "reg0_b0p", int(t6.register_cluster.debug_reg0[0].predicate), + "reg0_b1", int(t6.register_cluster.debug_reg0[1].payload), + "reg0_b1p", int(t6.register_cluster.debug_reg0[1].predicate), + "reg_rd_b0", int(t6.register_cluster.debug_reg_read[0].payload), + "reg_rd_b0p", int(t6.register_cluster.debug_reg_read[0].predicate), + "reg_rd_b1", int(t6.register_cluster.debug_reg_read[1].payload), + "reg_rd_b1p", int(t6.register_cluster.debug_reg_read[1].predicate), + flush=True) + if os.environ.get("CGRA_SKIP_BAD_RETURNS", "0") != "1" or \ + int(cpu_pkt.payload.data.payload) == expected_result: + assert int(cpu_pkt.payload.data.payload) == expected_result + assert int(cpu_pkt.payload.data.predicate) == 1 + debug_event = False + if debug_progress_enabled and \ + hasattr(th.dut, "debug_tile_elem_recv_opt_val"): + debug_event = cycle >= debug_from_cycle and any( + (int(th.dut.debug_tile_elem_recv_opt_val[tid]) and + int(th.dut.debug_tile_elem_recv_opt_op[tid]) in + (int(OPT_RET), int(OPT_GRT_PRED))) or + int(th.dut.debug_tile_to_ctrl_val[tid]) + for tid in debug_tile_ids) + if debug_progress_enabled and \ + ((cycle < 5) or + (cycle >= debug_from_cycle and debug_every > 0 and + cycle % debug_every == 0) or + debug_event): + cur_cmd = -1 + cur_dst = -1 + cur_src = -1 + cur_data = 0 + if int(th.src_ctrl_pkt.send.val): + cur_pkt = th.src_ctrl_pkt.send.msg + cur_cmd = int(cur_pkt.payload.cmd) + cur_dst = int(cur_pkt.dst) + cur_src = int(cur_pkt.src) + cur_data = int(cur_pkt.payload.data.payload) + print("[debug]", + "cycle", cycle, + "src_idx", th.src_ctrl_pkt.idx, + "src_done", th.src_ctrl_pkt.done(), + "src_val", int(th.src_ctrl_pkt.send.val), + "src_rdy", int(th.src_ctrl_pkt.send.rdy), + "cmd", cur_cmd, + "src", cur_src, + "dst", cur_dst, + "data", cur_data, + "preload", int(th.preload_sent_count), + "drain", int(th.preload_drain_count), + "draining", int(th.preload_draining), + "complete", int(th.complete_count), + flush=True) + if hasattr(th.dut, "debug_tile_times"): + tile_parts = [] + for tid in debug_tile_ids: + tile_parts.append( + f"t{tid}:tm{int(th.dut.debug_tile_times[tid])}" + f"a{int(th.dut.debug_tile_ctrl_addr[tid])}" + f"op{int(th.dut.debug_tile_op[tid])}" + f"p{int(th.dut.debug_tile_prologue_count_fu[tid])}" + f"v{int(th.dut.debug_tile_send_ctrl_val[tid])}" + f"r{int(th.dut.debug_tile_send_ctrl_rdy[tid])}" + f"s{int(th.dut.debug_tile_start[tid])}" + f"tc{int(th.dut.debug_tile_to_ctrl_val[tid])}" + f"/{int(th.dut.debug_tile_to_ctrl_cmd[tid])}" + f"/{int(th.dut.debug_tile_to_ctrl_data[tid])}" + f".{int(th.dut.debug_tile_to_ctrl_pred[tid])}" + ) + print("[debug_tiles]", "cycle", cycle, " | ".join(tile_parts), + flush=True) + if hasattr(th.dut, "debug_tile_elem_recv_opt_val"): + elem_parts = [] + for tid in debug_elem_tile_ids: + in_parts = [] + for i in range(num_fu_inports): + in_parts.append( + f"i{i}{int(th.dut.debug_tile_elem_recv_in_val[tid][i])}" + f"{int(th.dut.debug_tile_elem_recv_in_rdy[tid][i])}" + f":{int(th.dut.debug_tile_elem_recv_in_data[tid][i])}" + f".{int(th.dut.debug_tile_elem_recv_in_pred[tid][i])}" + ) + out_parts = [] + for i in range(num_fu_outports): + out_parts.append( + f"o{i}{int(th.dut.debug_tile_elem_send_out_val[tid][i])}" + f"{int(th.dut.debug_tile_elem_send_out_rdy[tid][i])}" + f":{int(th.dut.debug_tile_elem_send_out_data[tid][i])}" + f".{int(th.dut.debug_tile_elem_send_out_pred[tid][i])}" + ) + elem_parts.append( + f"t{tid}:eop{int(th.dut.debug_tile_elem_recv_opt_op[tid])}" + f"v{int(th.dut.debug_tile_elem_recv_opt_val[tid])}" + f"r{int(th.dut.debug_tile_elem_recv_opt_rdy[tid])}" + f"fi{int(th.dut.debug_tile_elem_recv_opt_fu_in0[tid])}" + f",{int(th.dut.debug_tile_elem_recv_opt_fu_in1[tid])}" + f"vf{int(th.dut.debug_tile_elem_recv_opt_vfp[tid])}" + f"l{int(th.dut.debug_tile_elem_recv_opt_is_last[tid])}" + f"rv{int(th.dut.debug_tile_elem_selected_reached_vf[tid])}" + f"vc{int(th.dut.debug_tile_elem_selected_vf_counter[tid])}" + f"[{','.join(in_parts)}]" + f"[{','.join(out_parts)}]" + f"sc{int(th.dut.debug_tile_elem_send_ctrl_val[tid])}" + f"{int(th.dut.debug_tile_elem_send_ctrl_rdy[tid])}" + f"/{int(th.dut.debug_tile_elem_send_ctrl_cmd[tid])}" + f"/{int(th.dut.debug_tile_elem_send_ctrl_data[tid])}" + f".{int(th.dut.debug_tile_elem_send_ctrl_pred[tid])}" + ) + print("[debug_elem]", "cycle", cycle, " | ".join(elem_parts), + flush=True) + if hasattr(th.dut, "debug_tile_route_recv_val"): + port_names = ["N", "S", "W", "E"] + route_parts = [] + for tid in debug_route_tile_ids: + recv_parts = [] + send_parts = [] + for i, pname in enumerate(port_names): + recv_parts.append( + f"{pname}{int(th.dut.debug_tile_route_recv_val[tid][i])}" + f"{int(th.dut.debug_tile_route_recv_rdy[tid][i])}" + f":{int(th.dut.debug_tile_route_recv_data[tid][i])}" + f".{int(th.dut.debug_tile_route_recv_pred[tid][i])}" + ) + send_parts.append( + f"{pname}{int(th.dut.debug_tile_send_val[tid][i])}" + f"{int(th.dut.debug_tile_send_rdy[tid][i])}" + f":{int(th.dut.debug_tile_send_data[tid][i])}" + f".{int(th.dut.debug_tile_send_pred[tid][i])}" + ) + local_parts = [] + write_parts = [] + reg_parts = [] + for i in range(num_fu_inports): + local_parts.append( + f"l{i}{int(th.dut.debug_tile_route_local_val[tid][i])}" + f"{int(th.dut.debug_tile_route_local_rdy[tid][i])}" + f":{int(th.dut.debug_tile_route_local_data[tid][i])}" + f".{int(th.dut.debug_tile_route_local_pred[tid][i])}" + ) + write_parts.append( + f"w{i}{int(th.dut.debug_tile_reg_write_val[tid][i])}" + f":{int(th.dut.debug_tile_reg_write_data[tid][i])}" + f".{int(th.dut.debug_tile_reg_write_pred[tid][i])}" + ) + if hasattr(th.dut, "debug_tile_reg0_data"): + reg_parts.append( + f"b{i}rd:{int(th.dut.debug_tile_reg_read_data[tid][i])}" + f".{int(th.dut.debug_tile_reg_read_pred[tid][i])}" + f"$0:{int(th.dut.debug_tile_reg0_data[tid][i])}" + f".{int(th.dut.debug_tile_reg0_pred[tid][i])}" + ) + route_parts.append( + f"t{tid}:rin[{','.join(recv_parts)}]" + f"rout[{','.join(send_parts)}]" + f"loc[{','.join(local_parts)}]" + f"wr[{','.join(write_parts)}]" + f"reg[{','.join(reg_parts)}]" + ) + print("[debug_route]", "cycle", cycle, " | ".join(route_parts), + flush=True) + if os.environ.get("CGRA_DEBUG_MEM", "0") == "1": + dm = th.dut.data_mem + bank0 = dm.memory_wrapper[0] + print("[debug_mem]", + "cycle", cycle, + "raddr3", int(dm.recv_raddr[3].msg), + "raddr3v", int(dm.recv_raddr[3].val), + "raddr3r", int(dm.recv_raddr[3].rdy), + "rdpkt3", str(dm.rd_pkt[3]), + "readxb0v", int(dm.read_crossbar.send[0].val), + "readxb0r", int(dm.read_crossbar.send[0].rdy), + "readxb0", str(dm.read_crossbar.send[0].msg), + "bank0rdv", int(bank0.recv_rd.val), + "bank0rdr", int(bank0.recv_rd.rdy), + "bank0rd", str(bank0.recv_rd.msg), + "bank0sendv", int(bank0.send.val), + "bank0sendr", int(bank0.send.rdy), + "bank0send", str(bank0.send.msg), + "resp3v", int(dm.response_crossbar.send[3].val), + "resp3r", int(dm.response_crossbar.send[3].rdy), + "resp3", str(dm.response_crossbar.send[3].msg), + "send3v", int(dm.send_rdata[3].val), + "send3r", int(dm.send_rdata[3].rdy), + "send3", str(dm.send_rdata[3].msg), + "mem6", str(bank0.memory.regs[6]), + "mem7", str(bank0.memory.regs[7]), + "mem8", str(bank0.memory.regs[8]), + flush=True) + if use_verilator: + if heartbeat_enabled and cycle and cycle % 1000 == 0: + print(f"[heartbeat] cycle={cycle}/{MAX_CYCLES}", flush=True) + if th.done(): + print(f"\n=== SIMULATION DONE at cycle {cycle} ===") + break + continue + # Collect state for all active tiles + cur_state = {} + for tid, t in active_tiles.items(): + cq = t.const_mem + cm = t.ctrl_mem + raddr = int(cm.reg_file.raddr[0]) + rdcur = int(cq.rd_cur) + cur_state[tid] = (raddr, rdcur) + + if heartbeat_enabled and cycle and cycle % 100 == 0: + max_times = max(int(t.ctrl_mem.times) for t in active_tiles.values()) + print(f"[heartbeat] cycle={cycle} max_times={max_times}/{kTotalCtrlSteps}", + flush=True) + + any_changed = any(cur_state[tid] != prev_state[tid] for tid in active_tiles) + kernel_started = any( + int(t.ctrl_mem.start_iterate_ctrl) or int(t.ctrl_mem.times) + for t in active_tiles.values() + ) + + if any_changed or not kernel_started: + stall_count = 0 + else: + stall_count += 1 + if stall_count >= 50: + print(f"\n=== EARLY DEADLOCK DETECTED at cycle {cycle} (stalled {stall_count} cycles) ===") + break + + if progress_enabled and \ + (any_changed or (cycle < 5) or (cycle % 200 == 0)): + parts = [] + for tid, t in sorted(active_tiles.items()): + cm = t.ctrl_mem + cq = t.const_mem + raddr = int(cm.reg_file.raddr[0]) + times = int(cm.times) + start = int(cm.start_iterate_ctrl) + op = int(cm.send_ctrl.msg.operation) if start else 0 + e = int(t.element_done) + r = int(t.routing_crossbar_done) + f = int(t.fu_crossbar_done) + rdcur = int(cq.rd_cur) + cv = int(cq.send_const.val) + cr = int(cq.send_const.rdy) + cc = int(t.const_consumed) if hasattr(t, "const_consumed") else 0 + pf = int(cm.prologue_count_outport_fu) + changed = "*" if cur_state[tid] != prev_state[tid] else " " + parts.append(f"t{tid}{changed}a{raddr}:0x{op:02x}t{times}e{e}r{r}f{f}q{rdcur}c{cv}{cr}{cc}p{pf}") + print(f"[cyc={cycle:4d}] {' | '.join(parts)}") + + prev_state = cur_state + + if th.done(): + print(f"\n=== SIMULATION DONE at cycle {cycle} ===") + break + + if not th.done() and use_verilator: + print(f"\n=== TIMEOUT after {MAX_CYCLES} cycles ===") + + if not th.done() and not use_verilator: + print(f"\n=== DEADLOCK after {MAX_CYCLES} cycles ===") + # Print final state of all active tiles (1D flat index in CgraRTL) + tile_map = { + "tile2": th.dut.tile[2], "tile3": th.dut.tile[3], + "tile5": th.dut.tile[5], "tile6": th.dut.tile[6], + "tile7": th.dut.tile[7], "tile9": th.dut.tile[9], + "tile10": th.dut.tile[10], "tile11": th.dut.tile[11], + } + dir_names = ["N", "S", "W", "E"] + for name, t in tile_map.items(): + cq = t.const_mem + cm = t.ctrl_mem + raddr = int(cm.reg_file.raddr[0]) + times = int(cm.times) + op = int(cm.send_ctrl.msg.operation) + rdcur = int(cq.rd_cur) + wrcur = int(cq.wr_cur) + cval = int(cq.send_const.msg.payload) + cv = int(cq.send_const.val) + cr = int(cq.send_const.rdy) + cp = int(cq.ctrl_proceed) + ed = int(t.element_done) + rd = int(t.routing_crossbar_done) + fd = int(t.fu_crossbar_done) + print(f" {name}: raddr={raddr} op=0x{op:02x} times={times} | rd_cur={rdcur} wr_cur={wrcur} const={cval}.v={cv}.rdy={cr} | " + f"ctrl_proceed={cp} e={ed} r={rd} f={fd}") + # Show channel states (recv_data val/rdy on each port) + ch_parts = [] + for pi in range(4): + rv = int(t.recv_data[pi].val) + rr = int(t.recv_data[pi].rdy) + sv = int(t.send_data[pi].val) + sr = int(t.send_data[pi].rdy) + ch_parts.append(f"{dir_names[pi]}:rv{rv}rr{rr}sv{sv}sr{sr}") + print(f" channels: {' | '.join(ch_parts)}") + # Show routing crossbar outport config and prologue counters + rxbar = t.routing_crossbar + rxbar_out = [] + for oi in range(len(rxbar.crossbar_outport)): + rxbar_out.append(int(rxbar.crossbar_outport[oi])) + print(f" routing_xbar outport={rxbar_out} recv_opt.val={int(rxbar.recv_opt.val)} recv_opt.rdy={int(rxbar.recv_opt.rdy)}") + # Prologue counters for current addr + prologue_parts = [] + for inp in range(4): + pc = int(rxbar.prologue_counter[raddr][inp]) + pcfg = int(rxbar.prologue_count_wire[raddr][inp]) + if pcfg > 0: + prologue_parts.append(f"in{inp}:{pc}/{pcfg}") + if prologue_parts: + print(f" routing prologue[addr{raddr}]: {' '.join(prologue_parts)}") + consts = [] + for idx in range(wrcur): + consts.append(int(cq.reg_file.regs[idx].payload)) + if consts: + print(f" const_queue: {consts}") + + if trace_enabled: + close_trace_logger() + + cycles = cycle + 1 + print("\n\n\ncycles: ", cycles) + assert th.done(), f"conv4x4 did not complete in {cycles} cycles" + + +def test_homogeneous_4x4_conv_combinational_mem_access(cmdline_opts): + sim_conv(cmdline_opts, mem_access_is_combinational = True) diff --git a/validation/script_generator.py b/validation/script_generator.py index b91b84bb..554464aa 100644 --- a/validation/script_generator.py +++ b/validation/script_generator.py @@ -2,7 +2,7 @@ ========================================================================== script_generator.py ========================================================================== -A translator to translate yaml-formatted IR generated by compiler to the +A translator to translate yaml-formatted IR generated by compiler to the packets used in VectorCGRA Author : Bohan Cui @@ -10,7 +10,7 @@ """ ### Usage: -# 1. create factory instance with the types +# 1. create factory instance with the types # (for the spec of the types, please refer to e.g. cgra/test/CgraRTL_fir_test.py) # ``` # script_factory = ScriptFactory( @@ -46,6 +46,9 @@ import os import yaml +# Set to True for verbose debug output during packet generation +VERBOSE_PKT_GEN = os.environ.get('VERBOSE_PKT_GEN', '0') == '1' + # Add project root to path to allow imports from lib # Get the absolute path of this file _script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -75,13 +78,19 @@ "FMUL": OPT_FMUL, "FDIV": None, # ? "OR": OPT_OR, + "AND": OPT_AND, "NOT": OPT_NOT, - "ICMP": None, # ? + "ICMP_EQ": OPT_EQ, # ? + "ICMP_SGE": OPT_GTE, + "ICMP_ULT": OPT_LT, + "ICMP_SGT": OPT_GT, + "ICMP_SLT": OPT_LT, "FCMP": None, # ? "SEL": OPT_SEL, "CAST": None, # ? "SEXT": OPT_PAS, # no sext, just a fake one. - "ZEXT": None, # ? + "ZEXT": OPT_PAS, # no zext, just a fake one (pass-through). + "CAST_TRUNC": OPT_PAS, # truncation treated as pass-through. "SHL": OPT_LLS, "VFMUL": None, # ? "FADD_FADD": None, #? @@ -94,21 +103,49 @@ "GRANT_ONCE": OPT_GRT_ONCE, #? "PHI": OPT_PHI, "LOOP_CONTROL": None, #? - "PHI_CONST": OPT_PHI_CONST, - + "PHI_CONST": OPT_PHI_CONST, + "PHI_START": OPT_PHI_START, + + "GEP": OPT_ADD, # By now, we just support 2 op GEP and it is equivalent to ADD (base + index) + "GEP_2D": OPT_GEP_2D, # special GEP for 2D array access, with 3 operands (base, idx1, idx2) and idx1/idx2 are multiplied with different stride. + "RETURN": OPT_RET, + "RETURN_VALUE": OPT_RET, + "RETURN_VOID": OPT_RET_VOID, "LDD": OPT_LD, - + "LOAD": OPT_LD, + "STORE": OPT_STR, + "NE": OPT_NE, "MUL_ADD": OPT_MUL_ADD, "DATA_MOV": OPT_PAS - + } yaml_to_VectorCGRA_map_const = { + "CONSTANT": OPT_CONST, "NE": OPT_NE_CONST, "ADD": OPT_ADD_CONST, "MUL_ADD": OPT_MUL_CONST_ADD, + "MUL": OPT_MUL_CONST, + "SUB": OPT_SUB_CONST, + "DIV": OPT_DIV_CONST, + "REM": OPT_REM_CONST, + "GEP": OPT_ADD_CONST, # For 2-op GEP with const base: base(const) + index(in0) + "GEP_2D": OPT_GEP_2D_CONST, + # Note: 3-op GEP with const base uses OPT_GEP_2D_CONST (handled dynamically) + "ICMP_EQ": OPT_EQ_CONST, + "ICMP_SGE": OPT_GTE_CONST, + "ICMP_ULT": OPT_LT_CONST, + "ICMP_SGT": OPT_GT_CONST, + "ICMP_SLT": OPT_LT_CONST, + "AND": OPT_AND_CONST, + "OR": OPT_OR_CONST, + "CONSTANT": OPT_CONST, + "SHL": OPT_LLS_CONST, + + "GRANT_ONCE": OPT_GRT_ONCE_CONST, + "STORE": OPT_STR_CONST, # default; overridden for STORE with const data } @@ -122,17 +159,46 @@ def _type(Operand): else: return 'IMM' +def direction_to_idx(direction): + if direction == 'NORTH': + return 0 + elif direction == 'SOUTH': + return 1 + elif direction == 'WEST': + return 2 + elif direction == 'EAST': + return 3 + else: + raise ValueError("Invalid direction: ", direction) + def _is_take_up_fu_operation(operation): - if operation['opcode'] == 'MOV' or operation['opcode'] == 'DATA_MOV': + if operation['opcode'] == 'MOV' or operation['opcode'] == 'DATA_MOV' or operation['opcode'] == 'CTRL_MOV': if len(operation['src_operands']) != 1: raise ValueError("MOV operation must have exactly one source operand") + elif _type(operation['src_operands'][0]) == 'PORT': + return False # By now, only Port -> Port and Port -> Reg are not take up fu operations elif _type(operation['src_operands'][0]) == 'REG': - return True - else: return False + else: + # REG -> PORT only: can be handled by FU output crossbar routing + # without occupying the FU, so treat as not-take-up-fu. + dst_operands = operation.get('dst_operands', []) + if dst_operands and all(_type(d) == 'PORT' for d in dst_operands): + return False + return True else: return True - + +def _take_up_fu_operation_idx_of(operations: list): + # check if there is only < 1 take up fu operation, if 1, return the idx, if > 1, raise error, if 0, return -1 + take_up_fu_operation_idx = -1 + for idx, operation in enumerate(operations): + if _is_take_up_fu_operation(operation): + if take_up_fu_operation_idx != -1: + raise ValueError("Only one take up fu operation is allowed, panic in instruction ", operations) + take_up_fu_operation_idx = idx + return take_up_fu_operation_idx + def _reg_cluster_no_of(operand): # start from 1 impl = operand['operand'] if impl[0] == "$": @@ -147,34 +213,40 @@ def _reg_cluster_intra_index_of(operand): else: raise ValueError("Operand is not a register") +def print_instruction(instruction): + if VERBOSE_PKT_GEN: print(f"Instruction: {[operation['opcode'] for operation in instruction['operations']]}") + FROM_NOWHERE = 0 FROM_PORT = 1 FROM_FU = 2 FROM_CONSTANT_QUEUE = 3 # not used by now -OPR_FROM_PORT = 0 -OPR_FROM_REGISTER = 1 +OPR_FROM_PORT = 0 # read_reg_towards = 0 = READ_TOWARDS_NOTHING +OPR_FROM_REGISTER = 1 # read_reg_towards = 1 = READ_TOWARDS_FU +OPR_TOWARDS_ROUTING = 2 # read_reg_towards = 2 = READ_TOWARDS_ROUTING_XBAR +OPR_TOWARDS_BOTH = 3 # read_reg_towards = 3 = READ_TOWARDS_BOTH class InstructionSignals: # to make signal of single instruction - def __init__(self, + def __init__(self, # input id_, operations, opcode_in_EIR, ctrl_addr, # types - IntraCgraPktType, - CgraPayloadType, - TileInType, - FuOutType, - CMD_CONFIG_input, + IntraCgraPktType, + CgraPayloadType, + TileInType, + FuOutType, + CMD_CONFIG_input, CtrlType, FuInType, B1Type, B2Type, RegIdxType, - CtrlAddrType): + CtrlAddrType, + accumulate_add_to_src_reg=False): # types self.IntraCgraPktType = IntraCgraPktType self.CgraPayloadType = CgraPayloadType @@ -187,93 +259,225 @@ def __init__(self, self.B2Type = B2Type self.RegIdxType = RegIdxType self.CtrlAddrType = CtrlAddrType - + self.accumulate_add_to_src_reg = accumulate_add_to_src_reg + # inputs self.id_ = id_ self.operations = operations self.OpCode = opcode_in_EIR self.ctrl_addr = ctrl_addr - + # States self.TileInParams = [-1, -1, -1, -1, -1, -1, -1, -1] self.FuOutParams = [-1, -1, -1, -1, -1, -1, -1, -1] - self.read_from_reg = [-1, -1, -1, -1] - self.read_from_reg_idx = [-1, -1, -1, -1] + self.read_reg_towards_fu = [-1, -1, -1, -1] + self.read_reg_towards_xbar = [-1, -1, -1, -1] + self.operand_from = [-1, -1, -1, -1] + self.read_towards_reg_idx = [-1, -1, -1, -1] self.write_to_reg = [-1, -1, -1, -1] self.write_to_reg_idx = [-1, -1, -1, -1] self.shuffle_fu_operand_input_index = [-1, -1, -1, -1] - + def buildCtrlPkt(self) -> list: # return the const needed to put into the const queue # you must call buildCtrlPkt before makeCtrlPkt, and for a Tile's signal, you must call buildCtrlPkt in order. try: - take_up_fu_operation_idx = -1 - for idx, operation in enumerate(self.operations): - if _is_take_up_fu_operation(operation): - if take_up_fu_operation_idx != -1: - raise ValueError("Only one take up fu operation is allowed") - take_up_fu_operation_idx = idx - - # TODO: fix logic here - - take_up_fu_operation = self.operations[take_up_fu_operation_idx] - has_const = False - # if has src_operands, check if has const - try: - src_operands = take_up_fu_operation['src_operands'] - except Exception as e: - src_operands = [] - for src_operand in src_operands: - if _type(src_operand) == 'IMM': - has_const = True - break - - if take_up_fu_operation['opcode'] == 'PHI_CONST' or take_up_fu_operation['opcode'] == 'CONSTANT' or take_up_fu_operation['opcode'] == 'GRANT_ONCE': - has_const = False # PHI_CONST and CONSTANT are special. - - if has_const: - self.opCode = yaml_to_VectorCGRA_map_const[self.operations[take_up_fu_operation_idx]['opcode']] + + # 1. Find take up fu operation + take_up_fu_operation_idx = _take_up_fu_operation_idx_of(self.operations) + + if take_up_fu_operation_idx == -1: + if VERBOSE_PKT_GEN: print("No take up fu operation found, just take the first.") + take_up_fu_operation_idx = 0 + take_up_fu_operation = "NAH" + self.OpCode = OPT_NAH else: - self.opCode = yaml_to_VectorCGRA_map[self.operations[take_up_fu_operation_idx]['opcode']] + if VERBOSE_PKT_GEN: print("Take_up_fu_operation: ", self.operations[take_up_fu_operation_idx]) + take_up_fu_operation = self.operations[take_up_fu_operation_idx] + # only take up fu can be const + has_const = False + # if has src_operands, check if has const + try: + src_operands = take_up_fu_operation['src_operands'] + except Exception as e: + src_operands = [] + for src_operand in src_operands: + if _type(src_operand) == 'IMM': + has_const = True + break + + #if take_up_fu_operation['opcode'] == 'PHI_CONST' or take_up_fu_operation['opcode'] == 'CONSTANT': + #has_const = False # PHI_CONST and CONSTANT are special. + + if has_const: + self.OpCode = yaml_to_VectorCGRA_map_const[self.operations[take_up_fu_operation_idx]['opcode']] + # Special handling for STORE with const: + # YAML STORE format is [data, addr]. After swap → [addr, data]. + # If the IMM is the DATA (position 0 in YAML), use OPT_STR_DATA_CONST. + # If the IMM is the ADDR (position 1 in YAML), use OPT_STR_CONST. + if take_up_fu_operation['opcode'] == 'STORE': + if len(src_operands) >= 2 and _type(src_operands[0]) == 'IMM': + # Data is const (position 0 in YAML → position 1 after swap) + self.OpCode = OPT_STR_DATA_CONST + else: + # Address is const (position 1 in YAML → position 0 after swap) + self.OpCode = OPT_STR_CONST + # Special handling for 3-operand GEP with const base (2D array access): + # If GEP has 3 src_operands and one is an IMM (const base), + # use OPT_GEP_2D_CONST instead of OPT_ADD_CONST. + if take_up_fu_operation['opcode'] == 'GEP': + non_imm_count = sum(1 for op in src_operands if _type(op) != 'IMM') + if non_imm_count >= 2: + self.OpCode = OPT_GEP_2D_CONST + else: + self.OpCode = yaml_to_VectorCGRA_map[self.operations[take_up_fu_operation_idx]['opcode']] const_operands = [] - for operation in self.operations: # for each operation in the instruction - - print("Working on operation: ", operation) - + # Two-pass processing: First handle non-FU operations (to reserve + # their fixed lanes/TileInParams), then handle FU operations (which + # pick from remaining available lanes). + ordered_operations = sorted( + self.operations, + key=lambda op: 0 if not _is_take_up_fu_operation(op) else 1) + + for operation in ordered_operations: # for each operation in the instruction + + if VERBOSE_PKT_GEN: print("\n Operation: ", operation) + operation_opcode = operation['opcode'] try: - src_operands = operation['src_operands'] + src_operands = operation['src_operands'].copy() + if operation_opcode == 'STORE' and len(src_operands) >= 2: + # HW expects address in0 and data in1, but YAML gives [data, addr] + src_operands[0], src_operands[1] = src_operands[1], src_operands[0] except Exception as e: src_operands = [] try: - dst_operands = operation['dst_operands'] + dst_operands = list(operation['dst_operands']) except Exception as e: dst_operands = [] - + # find all the const for index, src_operand in enumerate(src_operands): if _type(src_operand) == 'IMM': - const_operands.append(src_operand) + const_operands.append((src_operand, operation["time_step"])) # delete it from the src_operands since it is implicit in vectorCGRA del src_operands[index] - + + if self.accumulate_add_to_src_reg and operation_opcode == 'ADD': + reg_src_operands = [ + operand for operand in src_operands + if _type(operand) == 'REG' + ] + has_reg_dst = any(_type(operand) == 'REG' + for operand in dst_operands) + if reg_src_operands and not has_reg_dst: + dst_operands.append(reg_src_operands[0]) + + # for not_take_up_fu_operation + if not _is_take_up_fu_operation(operation): + + if len(src_operands) != 1: + raise ValueError(f"Not take up fu operation {operation} must have exactly one source operand.") + + src_operand = src_operands[0] + + # REG -> PORT: route via FU output crossbar (merge with + # the take-up-fu operation's output routing). + if _type(src_operand) == 'REG': + intra_index = _reg_cluster_intra_index_of(src_operand) + cluster_no = _reg_cluster_no_of(src_operand) + + # check collision + if self.read_reg_towards_xbar[cluster_no - 1] != -1 and self.read_towards_reg_idx[cluster_no - 1] != intra_index: + # another register calls for sending to the FU + raise ValueError(f"Collision when reading from register in read_reg_towards_fu, when translate the operation {operation} to VectorCGRA") + + if self.operand_from[cluster_no - 1] != -1 and self.operand_from[cluster_no - 1] == OPR_FROM_PORT: + raise ValueError(f"The register operand overwrite the port operand in lane {cluster_no -1}") + for index, dst_operand in enumerate(dst_operands): + if _type(dst_operand) == 'PORT': + if dst_operand['operand'] == 'NORTH': + port_out_xbar_idx = 0 + elif dst_operand['operand'] == 'SOUTH': + port_out_xbar_idx = 1 + elif dst_operand['operand'] == 'WEST': + port_out_xbar_idx = 2 + elif dst_operand['operand'] == 'EAST': + port_out_xbar_idx = 3 + if self.TileInParams[port_out_xbar_idx] != -1: + # another source want to send to the direction + raise ValueError(f"Collision in writing to port {dst_operand} in FuOutParams (REG->PORT), when translate the operation {operation} to VectorCGRA") + self.TileInParams[port_out_xbar_idx] = cluster_no + 4 # directly route from the register cluster to the port, no need to shuffle to FU first + self.read_reg_towards_xbar[cluster_no - 1] = 1 + self.read_towards_reg_idx[cluster_no - 1] = intra_index + self.operand_from[cluster_no - 1] = OPR_FROM_REGISTER + else: + raise ValueError(f"REG->non-PORT dst {dst_operand} classified as not take-up-fu, when translate the operation {operation} to VectorCGRA") + continue + + elif _type(src_operand) == 'PORT': + src_direction_idx = direction_to_idx(src_operand['operand']) + + for index, dst_operand in enumerate(dst_operands): + if _type(dst_operand) == 'REG': + cluster_no = _reg_cluster_no_of(dst_operand) + intra_index = _reg_cluster_intra_index_of(dst_operand) + # set the TileInParams + if self.TileInParams[cluster_no + 4 - 1] != -1 and self.TileInParams[cluster_no + 4 - 1] != src_direction_idx + 1: + raise ValueError(f"Collision when reading from port in TileInParams, when translate the operation {operation} to VectorCGRA") + self.TileInParams[cluster_no + 4 - 1] = src_direction_idx + 1 + + if self.write_to_reg_idx[cluster_no - 1] != -1 and self.write_to_reg_idx[cluster_no - 1] != intra_index: + raise ValueError(f"Collision when writing to register in write_to_reg_idx, when translate the operation {operation} to VectorCGRA") + self.write_to_reg[cluster_no - 1] = FROM_PORT + self.write_to_reg_idx[cluster_no - 1] = intra_index + elif _type(dst_operand) == 'PORT': + if dst_operand['operand'] == 'NORTH': + port_out_xbar_idx = 0 + elif dst_operand['operand'] == 'SOUTH': + port_out_xbar_idx = 1 + elif dst_operand['operand'] == 'WEST': + port_out_xbar_idx = 2 + elif dst_operand['operand'] == 'EAST': + port_out_xbar_idx = 3 + if self.TileInParams[port_out_xbar_idx] != -1 and self.TileInParams[port_out_xbar_idx] != src_direction_idx + 1: + raise ValueError(f"Collision in writing to port {dst_operand} in FuOutParams, when translate the operation {operation} to VectorCGRA") + self.TileInParams[port_out_xbar_idx] = src_direction_idx + 1 # 1,2,3,4 for N,S,W,E + else: + raise ValueError(f"Unsupported type of dst operand {dst_operand}, when translate the operation {operation} to VectorCGRA") + continue + + + # reorder the src operands since register has high priority (cause reg can't be shuffled but port can be shuffled) + # TODO: not complete, actually should also take dst into account + reordered_src_operands = {} for index, src_operand in enumerate(src_operands): if _type(src_operand) == 'REG': - print(f">>> index {index} is REG") + reordered_src_operands[index] = src_operand + for index, src_operand in enumerate(src_operands): + if not _type(src_operand) == 'REG': + reordered_src_operands[index] = src_operand + + for index, src_operand in reordered_src_operands.items(): + if _type(src_operand) == 'REG': + if VERBOSE_PKT_GEN: print(f">>> index {index} is REG") cluster_no = _reg_cluster_no_of(src_operand) intra_index = _reg_cluster_intra_index_of(src_operand) # Check if cluster_no is within valid range (1 to num_fu_inports) - if cluster_no < 1 or cluster_no > len(self.read_from_reg_idx): - raise ValueError(f"Register cluster number {cluster_no} is out of range. Valid range is 1 to {len(self.read_from_reg_idx)} for register {src_operand['operand']} in operation {operation}") + if cluster_no < 1 or cluster_no > len(self.read_towards_reg_idx): + raise ValueError(f"Register cluster number {cluster_no} is out of range. Valid range is 1 to {len(self.read_towards_reg_idx)} for register {src_operand['operand']} in operation {operation}") # Check if intra_index is within valid range (0 to REG_CLUSTER_SIZE-1) if intra_index < 0 or intra_index >= REG_CLUSTER_SIZE: raise ValueError(f"Register intra index {intra_index} is out of range. Valid range is 0 to {REG_CLUSTER_SIZE-1} for register {src_operand['operand']} in operation {operation}") - if self.read_from_reg_idx[cluster_no - 1] != -1 and self.read_from_reg_idx[cluster_no - 1] != intra_index: + if self.read_reg_towards_fu[cluster_no - 1] != -1 and self.read_towards_reg_idx[cluster_no - 1] != intra_index: raise ValueError(f"Collision when reading from register in read_from_reg_idx, when translate the operation {operation} to VectorCGRA") - self.read_from_reg[cluster_no - 1] = OPR_FROM_REGISTER - self.read_from_reg_idx[cluster_no - 1] = intra_index + if self.operand_from[cluster_no - 1] != -1 and self.operand_from[cluster_no - 1] != OPR_FROM_REGISTER: + raise ValueError(f"Register operand overwrite the port operand in lane {cluster_no -1}, when translate the operation {operation} to VectorCGRA") + self.operand_from[cluster_no - 1] = OPR_FROM_REGISTER + self.read_towards_reg_idx[cluster_no - 1] = intra_index + self.read_reg_towards_fu[cluster_no - 1] = 1 if self.shuffle_fu_operand_input_index[index] != -1: raise ValueError(f"Collision when reading from register in shuffle_fu_operand_input_index, when translate the operation {operation} to VectorCGRA") self.shuffle_fu_operand_input_index[index] = cluster_no # shuffle the data to the correct inport of the FU from the register @@ -286,27 +490,29 @@ def buildCtrlPkt(self) -> list: # return the const needed to put into the const port_in_xbar_idx = 3 elif src_operand['operand'] == 'EAST': port_in_xbar_idx = 4 - + # find an available lane lane = -1 for i in range(4): - if self.read_from_reg[i] == -1: # if not set the selection, then it could be available + ok = self.operand_from[i] == -1 and self.TileInParams[i + 4] == -1 # The lane is completely empty + ok = ok or (self.operand_from[i] == OPR_FROM_PORT and self.TileInParams[i + 4] == port_in_xbar_idx) # The lane is already used by the same port, can be reused + if ok: lane = i break if lane == -1: raise ValueError(f"No available lane found when reading from port {src_operand} in TileInParams, when translate the operation {operation} to VectorCGRA") - if self.TileInParams[lane + 4] != -1: - raise ValueError(f"Collision in reading from port in TileInParams, when translate the operation {operation} to VectorCGRA") + else: + if VERBOSE_PKT_GEN: print(f"Lane {lane} available for port source {src_operand}") self.TileInParams[lane + 4] = port_in_xbar_idx - self.read_from_reg[lane] = OPR_FROM_PORT + self.operand_from[lane] = OPR_FROM_PORT # shuffle to the given fu input index if self.shuffle_fu_operand_input_index[index] != -1: - raise ValueError(f"Collision when reading from port in shuffle_fu_operand_input_index, when translate the operation {operation} to VectorCGRA") + raise ValueError(f"Collision when reading from port in shuffle_fu_operand_input_index {self.shuffle_fu_operand_input_index} (want to set opr {index} to {lane}), when translate the operation {operation} to VectorCGRA") self.shuffle_fu_operand_input_index[index] = lane + 1 - + if _type(src_operand) == 'IMM': - raise NotImplementedError("IMM src operand is not supported yet") - + raise NotImplementedError("IMM src should have been removed here") + for index, dst_operand in enumerate(dst_operands): if _type(dst_operand) == 'REG': cluster_no = _reg_cluster_no_of(dst_operand) @@ -335,24 +541,24 @@ def buildCtrlPkt(self) -> list: # return the const needed to put into the const port_out_xbar_idx = 3 if self.FuOutParams[port_out_xbar_idx] != -1: raise ValueError(f"Collision in writing to port {dst_operand} in FuOutParams, when translate the operation {operation} to VectorCGRA") - print(f">>> FuOutParams[{port_out_xbar_idx}] = {index + 1}") - self.FuOutParams[port_out_xbar_idx] = 1 # we do not support multiple results + if VERBOSE_PKT_GEN: print(f">>> FuOutParams[{port_out_xbar_idx}] = {index + 1}") + self.FuOutParams[port_out_xbar_idx] = 1 # we do not support multiple results else: raise ValueError(f"Unsupported type of dst operand {dst_operand}, when translate the operation {operation} to VectorCGRA") - + except Exception as e: print(f"Error in making ctrl pkt: {e}") raise e return None return const_operands - + def makeCtrlPkt(self): # make fu_in_code for idx, fu_in_code in enumerate(self.shuffle_fu_operand_input_index): if fu_in_code == -1: - self.shuffle_fu_operand_input_index[idx] = idx + 1 # 0 or idle inport of ALU? + self.shuffle_fu_operand_input_index[idx] = idx + 1 fu_in_code_made = [self.FuInType(x) for x in self.shuffle_fu_operand_input_index] # is it correct? - + # make TileIn for idx, tile_in_param in enumerate(self.TileInParams): if tile_in_param == -1: @@ -362,31 +568,38 @@ def makeCtrlPkt(self): if fu_out_param == -1: self.FuOutParams[idx] = 0 FuOut_made = [self.FuOutType(x) for x in self.FuOutParams] - + # made write reg from code for idx, write_to_reg in enumerate(self.write_to_reg): if write_to_reg == -1: self.write_to_reg[idx] = FROM_NOWHERE write_reg_from_made = [self.B2Type(x) for x in self.write_to_reg] - + for idx, write_to_reg_idx in enumerate(self.write_to_reg_idx): if write_to_reg_idx == -1: self.write_to_reg_idx[idx] = 0 write_reg_idx_made = [self.RegIdxType(x) for x in self.write_to_reg_idx] - + # make read reg from code - for idx, read_from_reg in enumerate(self.read_from_reg): - if read_from_reg == -1: - self.read_from_reg[idx] = OPR_FROM_PORT + read_towards = [-1, -1, -1, -1] + for idx in range(4): + if self.read_reg_towards_fu[idx] == 1 and self.read_reg_towards_xbar[idx] == 1: + read_towards[idx] = 3 # both + elif self.read_reg_towards_fu[idx] == 1: + read_towards[idx] = 1 # to fu + elif self.read_reg_towards_xbar[idx] == 1: + read_towards[idx] = 2 # to xbar + else: + read_towards[idx] = 0 # to nowhere # read_reg_towards uses 2-bit type (RegFromType): 0=nothing, 1=FU, 2=routing_xbar, 3=both - read_reg_towards_made = [self.B2Type(x) for x in self.read_from_reg] - - for idx, read_from_reg_idx in enumerate(self.read_from_reg_idx): - if read_from_reg_idx == -1: - self.read_from_reg_idx[idx] = 0 - read_reg_idx_made = [self.RegIdxType(x) for x in self.read_from_reg_idx] - + read_reg_towards_made = [self.B2Type(x) for x in read_towards] + + for idx, read_towards_reg_idx in enumerate(self.read_towards_reg_idx): + if read_towards_reg_idx == -1: + self.read_towards_reg_idx[idx] = 0 + read_reg_idx_made = [self.RegIdxType(x) for x in self.read_towards_reg_idx] + # make FuOut # CtrlType requires: operation, fu_in, routing_xbar_outport, fu_xbar_outport, # vector_factor_power, is_last_ctrl, write_reg_from, write_reg_idx, @@ -395,7 +608,7 @@ def makeCtrlPkt(self): pkt = self.IntraCgraPktType(0, self.id_, payload = self.CgraPayloadType(self.CMD_CONFIG_, ctrl_addr = self.CtrlAddrType(self.ctrl_addr), - ctrl = self.CtrlType(self.opCode, + ctrl = self.CtrlType(self.OpCode, fu_in_code_made, TileIn_made, FuOut_made, @@ -408,14 +621,14 @@ def makeCtrlPkt(self): class TileSignals: def __init__(self, - CtrlType, - IntraCgraPktType, - CgraPayloadType, - TileInType, - FuOutType, - CMD_CONFIG_input, - FuInType, - id_, + CtrlType, + IntraCgraPktType, + CgraPayloadType, + TileInType, + FuOutType, + CMD_CONFIG_input, + FuInType, + id_, loop_times, ii, instructions, @@ -431,7 +644,11 @@ def __init__(self, B2Type, RegIdxType, CtrlAddrType, - DataAddrType): + DataAddrType, + arg_map=None, + gep_stride=None, + use_time_step_ctrl_addr=False, + accumulate_add_to_src_reg=False): self.CtrlType = CtrlType self.IntraCgraPktType = IntraCgraPktType self.CgraPayloadType = CgraPayloadType @@ -449,133 +666,290 @@ def __init__(self, self.DataType = DataType self.CtrlAddrType = CtrlAddrType self.DataAddrType = DataAddrType + self.arg_map = arg_map if arg_map is not None else {} + self.gep_stride = gep_stride + self.use_time_step_ctrl_addr = use_time_step_ctrl_addr + self.accumulate_add_to_src_reg = accumulate_add_to_src_reg # constants self.CMD_CONST_ = CMD_CONST_input self.CMD_CONFIG_COUNT_PER_ITER_ = CMD_CONFIG_COUNT_PER_ITER_input self.CMD_CONFIG_TOTAL_CTRL_COUNT_ = CMD_CONFIG_TOTAL_CTRL_COUNT_input - + self.CMD_CONFIG_PROLOGUE_FU_ = CMD_CONFIG_PROLOGUE_FU_input self.CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR_ = CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR_input self.CMD_CONFIG_PROLOGUE_FU_CROSSBAR_ = CMD_CONFIG_PROLOGUE_FU_CROSSBAR_input - + self.CMD_LAUNCH_ = CMD_LAUNCH_input - - - def makeProloguePackets(self, instruction): - print(f"Making prologue packets for instruction {instruction}") - pkts = [] - pkts.append(self.makePrologueFUPackets(instruction)) # always prologue FU - take_up_fu_operation_idx = -1 - for idx, operation in enumerate(instruction['operations']): - if _is_take_up_fu_operation(operation): - if take_up_fu_operation_idx != -1: - raise ValueError("Only one take up fu operation is allowed") - take_up_fu_operation_idx = idx - take_up_fu_operation = None - if take_up_fu_operation_idx != -1: - take_up_fu_operation = instruction['operations'][take_up_fu_operation_idx] - # TODO: fix the logic for only having non-take up fu operation - else: - raise ValueError("No take up fu operation found") - try: - src_operands = take_up_fu_operation['src_operands'] - except Exception as e: - src_operands = [] - for src_operand in src_operands: - if _type(src_operand) == 'PORT': - print(f"src_operand is PORT, add Prologue. {src_operand}") - if src_operand['operand'] == 'NORTH': - routing_xbar_idx = 0 # here is from 0, strange. - elif src_operand['operand'] == 'SOUTH': - routing_xbar_idx = 1 - elif src_operand['operand'] == 'WEST': - routing_xbar_idx = 2 - elif src_operand['operand'] == 'EAST': - routing_xbar_idx = 3 - pkts.append(self.makePrologueRoutingCrossbarPackets(instruction, routing_xbar_idx)) - - try: - dst_operands = take_up_fu_operation['dst_operands'] - except Exception as e: - dst_operands = [] - for dst_operand in dst_operands: - if _type(dst_operand) == 'REG': - pkts.append(self.makePrologueFUCrossbarPackets(instruction)) - return pkts - - def makePhiConstProloguePackets(self, instruction): - print(f"Making phi const prologue packets for instruction {instruction}") + def sanitize(self, instruction): + # check 1: take_up_fu function <= 1 + take_up_fu_operation_idx = _take_up_fu_operation_idx_of(instruction['operations']) + + # check 2: prologue routing crossbar each port consistently either all ignore or not + routing_xbar_ports_if_prologued = {} for operation in instruction['operations']: - if operation['opcode'] == 'PHI_CONST': - phi_const_operation = operation - break - if phi_const_operation is None: - raise ValueError("No PHI_CONST operation found") - try: - src_operands = phi_const_operation['src_operands'] - except Exception as e: - src_operands = [] - - pkts = [] - - for src_operand in src_operands: - if _type(src_operand) == 'PORT': - if src_operand['operand'] == 'NORTH': - routing_xbar_idx = 0 - elif src_operand['operand'] == 'SOUTH': - routing_xbar_idx = 1 - elif src_operand['operand'] == 'WEST': - routing_xbar_idx = 2 - elif src_operand['operand'] == 'EAST': - routing_xbar_idx = 3 - pkts.append(self.makePrologueRoutingCrossbarPackets(instruction, routing_xbar_idx)) - - print(f"Phi const prologue packets: {pkts}") - return pkts - - - def makePrologueFUPackets(self, instruction): - return self.IntraCgraPktType(0, self.id_, - payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_FU_, ctrl_addr = self.CtrlAddrType(instruction['timestep'] % self.ii), - data = self.DataType(1, 1))) - def makePrologueRoutingCrossbarPackets(self, instruction, routing_xbar_idx): - return self.IntraCgraPktType(0, self.id_, - payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR_, ctrl_addr = self.CtrlAddrType(instruction['timestep'] % self.ii), + count = operation['invalid_iterations'] + + # for reg -> port + if not _is_take_up_fu_operation(operation): + if len(operation['src_operands']) != 1: + raise ValueError("Not take up fu operation must have exactly one source operand, panic in instruction ", instruction, " at operation ", operation) + src_operand = operation['src_operands'][0] + if _type(src_operand) == 'REG': + cluster_no = _reg_cluster_no_of(src_operand) + idd = f"R{cluster_no}" + # print (f"Check prologue for reg operand {src_operand['operand']} in operation {operation}, cluster_no {cluster_no}, idd {idd}") + if idd in routing_xbar_ports_if_prologued: + if routing_xbar_ports_if_prologued[idd] != count: + raise ValueError("Routing crossbar ports must be consistently either all ignore or not, panic in instruction ", instruction) + else: + routing_xbar_ports_if_prologued[idd] = count + + for src_operand in operation['src_operands']: + if _type(src_operand) == 'PORT': + port_name = src_operand['operand'] + if port_name in routing_xbar_ports_if_prologued: + # Take the max count for this port across all operations + routing_xbar_ports_if_prologued[port_name] = max(routing_xbar_ports_if_prologued[port_name], count) + else: + routing_xbar_ports_if_prologued[port_name] = count + + # check 3: all the time_steps are aligned with the index_per_ii + for operation in instruction['operations']: + if operation['time_step'] % self.ii != instruction['index_per_ii']: + raise ValueError("Time step is not aligned with index per ii, panic in instruction ", instruction, " at operation ", operation) + + return routing_xbar_ports_if_prologued + + + def makePrologueOperationPackets(self, operation): + res = [] + # make prologue packets for the operation + if _is_take_up_fu_operation(operation): + # prologue FU to ignore this packet + count = operation.get('invalid_iterations', 1) + res.append(self.makePrologueFUPackets(operation['time_step'], count)) + # also need to prologue the FU Output routing crossbar + res.append(self.makePrologueFUCrossbarPackets(operation['time_step'])) + + # src operands' prologues are not here, avoid repetition. + # for src_operand in operation['src_operands']: + # if _type(src_operand) == 'PORT': + # res.append(self.makePrologueRoutingCrossbarPackets(operation['time_step'], direction_to_idx(src_operand['operand']) + 1)) + + return res + + + def makePrologueFUPackets(self, timestep, count=1): + # make prologue FU packet with given prologue count + return self.IntraCgraPktType(0, self.id_, + payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_FU_, ctrl_addr = self.CtrlAddrType(timestep % self.ii), + data = self.DataType(count, 1))) + + def makePrologueRoutingCrossbarPackets(self, timestep, routing_xbar_idx, count=1): + return self.IntraCgraPktType(0, self.id_, + payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_ROUTING_CROSSBAR_, ctrl_addr = self.CtrlAddrType(timestep % self.ii), ctrl = self.CtrlType(routing_xbar_outport = [self.TileInType(routing_xbar_idx)] + [self.TileInType(0)] * 7), - data = self.DataType(1, 1))) - def makePrologueFUCrossbarPackets(self, instruction): - return self.IntraCgraPktType(0, self.id_, - payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_FU_CROSSBAR_, ctrl_addr = self.CtrlAddrType(instruction['timestep'] % self.ii), + data = self.DataType(count, 1))) + def makePrologueFUCrossbarPackets(self, timestep, count=1): + return self.IntraCgraPktType(0, self.id_, + payload = self.CgraPayloadType(self.CMD_CONFIG_PROLOGUE_FU_CROSSBAR_, ctrl_addr = self.CtrlAddrType(timestep % self.ii), ctrl = self.CtrlType(fu_xbar_outport = [self.FuOutType(0)] * 8), - data = self.DataType(1, 1))) + # WARN:by now, only support one result for each operation + data = self.DataType(count, 1))) + def expandInstructionsByTimeStep(self): + expanded = [] + for instruction in self.instructions: + grouped_ops = {} + for operation in instruction['operations']: + ctrl_addr = operation['time_step'] % self.ii + expanded_operation = dict(operation) + grouped_ops.setdefault(ctrl_addr, []).append(expanded_operation) + + for ctrl_addr in sorted(grouped_ops): + expanded_instruction = dict(instruction) + expanded_instruction['index_per_ii'] = ctrl_addr + expanded_instruction['operations'] = grouped_ops[ctrl_addr] + expanded.append(expanded_instruction) + return expanded + def makeTileSignals(self): consts = [] all_signals = [] all_instruction_signals = [] prologue_signals = [] has_addrs = [] - + instructions = self.expandInstructionsByTimeStep() \ + if self.use_time_step_ctrl_addr else self.instructions + + local_accumulator_regs = set() + if self.accumulate_add_to_src_reg: + for instruction in instructions: + for operation in instruction['operations']: + if operation.get('opcode') != 'ADD': + continue + src_operands = operation.get('src_operands', []) + dst_operands = operation.get('dst_operands', []) + if any(_type(operand) == 'REG' for operand in dst_operands): + continue + for operand in src_operands: + if _type(operand) == 'REG': + local_accumulator_regs.add(operand['operand']) + + if local_accumulator_regs: + used_regs = set() + for instruction in instructions: + for operation in instruction['operations']: + for operand in operation.get('src_operands', []): + if _type(operand) == 'REG': + used_regs.add(operand['operand']) + for operand in operation.get('dst_operands', []): + if _type(operand) == 'REG': + used_regs.add(operand['operand']) + + scratch_for_acc = {} + for reg_name in local_accumulator_regs: + cluster_base = (int(reg_name[1:]) // REG_CLUSTER_SIZE) * REG_CLUSTER_SIZE + for reg_idx in range(cluster_base + REG_CLUSTER_SIZE - 1, + cluster_base - 1, + -1): + candidate = f"${reg_idx}" + if candidate != reg_name and candidate not in used_regs: + scratch_for_acc[reg_name] = candidate + used_regs.add(candidate) + break + if reg_name not in scratch_for_acc: + raise ValueError( + f"No scratch register available to drain accumulator feedback for {reg_name}") + + filtered_instructions = [] + for instruction in instructions: + filtered_ops = [] + for operation in instruction['operations']: + src_operands = operation.get('src_operands', []) + dst_operands = operation.get('dst_operands', []) + is_external_acc_feedback = \ + operation.get('opcode') == 'DATA_MOV' and \ + len(src_operands) == 1 and \ + _type(src_operands[0]) == 'PORT' and \ + any(_type(dst) == 'REG' and + dst['operand'] in local_accumulator_regs + for dst in dst_operands) + if is_external_acc_feedback: + drain_operation = dict(operation) + drain_dsts = [] + for dst in dst_operands: + if _type(dst) == 'REG' and \ + dst['operand'] in local_accumulator_regs: + scratch_dst = dict(dst) + scratch_dst['operand'] = \ + scratch_for_acc[dst['operand']] + drain_dsts.append(scratch_dst) + else: + drain_dsts.append(dst) + drain_operation['dst_operands'] = drain_dsts + filtered_ops.append(drain_operation) + else: + filtered_ops.append(operation) + + if filtered_ops: + filtered_instruction = dict(instruction) + filtered_instruction['operations'] = filtered_ops + filtered_instructions.append(filtered_instruction) + instructions = filtered_instructions + + used_addrs = { + instruction['index_per_ii'] + for instruction in instructions + } + + shifted_ctrl_addrs = {} + + def is_shiftable_port_to_port(instruction): + for operation in instruction['operations']: + if operation.get('opcode') != 'DATA_MOV': + return False + src_operands = operation.get('src_operands', []) + dst_operands = operation.get('dst_operands', []) + if len(src_operands) != 1 or _type(src_operands[0]) != 'PORT': + return False + if not dst_operands or any(_type(dst) != 'PORT' for dst in dst_operands): + return False + return True + + for instruction in instructions: + ctrl_addr = instruction['index_per_ii'] + shifted_addr = (ctrl_addr + 1) % self.ii + if False and \ + int(self.id_) == 9 and \ + is_shiftable_port_to_port(instruction) and \ + shifted_addr not in used_addrs: + shifted_ctrl_addrs[id(instruction)] = shifted_addr + else: + shifted_ctrl_addrs[id(instruction)] = ctrl_addr + # build all the instruction signals and get all the const - for instruction in self.instructions: - if instruction['timestep'] >= self.ii: - prologue_signals.extend(self.makeProloguePackets(instruction)) - - has_phi_const = False # cope with special PHI_CONST operation + import sys + for instr_idx_dbg, instruction in enumerate(instructions): + # Suppress verbose per-instruction output for speed + pass + # do necessary sanitization + prologued_ports = self.sanitize(instruction) + ctrl_addr = shifted_ctrl_addrs[id(instruction)] + #print(prologued_ports) + + # only when the take_up_fu_op exists and no prologue, then no prologue + prologue_fu = False + max_invalid_iterations = 0 + for operation in instruction['operations']: - if operation['opcode'] == 'PHI_CONST': - has_phi_const = True - break - if has_phi_const: - prologue_signals.extend(self.makePhiConstProloguePackets(instruction)) - - has_addrs.append(instruction['timestep'] % self.ii) - + if operation['invalid_iterations'] > 0: + prologue_fu = True + max_invalid_iterations = max(max_invalid_iterations, operation['invalid_iterations']) + + for operation in instruction['operations']: + if _is_take_up_fu_operation(operation) and operation['invalid_iterations'] == 0: + prologue_fu = False + + # only non-taken prologue -> still need prologue FU + # only taken prologue -> need prologue FU + # +--------------------+----------+--------------+ + # | non-taken \ taken | prologue | not prologue | + # +--------------------+----------+--------------+ + # | prologue | yes | no | + # | not prologue | yes | no | + # +--------------------+----------+--------------+ + + if prologue_fu: + prologue_signals.append(self.makePrologueFUPackets(ctrl_addr, max_invalid_iterations)) + prologue_signals.append(self.makePrologueFUCrossbarPackets(ctrl_addr, max_invalid_iterations)) + + # add all routing crossbar prologues for all the operations (can not be op by op to avoid repetition) + for port_name, count in prologued_ports.items(): + if count > 0: + if port_name.startswith('R'): + cluster_no = int(port_name[1:]) + routing_xbar_idx = cluster_no + 4 - 1 + else: + routing_xbar_idx = direction_to_idx(port_name) + prologue_signals.append(self.makePrologueRoutingCrossbarPackets(ctrl_addr, routing_xbar_idx + 1, count)) + + # add an addtional prologue packet for PHI_CONST / PHI_START operation + for operation in instruction['operations']: + if operation['opcode'] == 'PHI_CONST' or operation['opcode'] == "PHI_START": + # only src1 will be prologued once, and only if it is a port, it needs a routing crossbar prologue + if _type(operation['src_operands'][1]) == 'PORT': + routing_xbar_idx = direction_to_idx(operation['src_operands'][1]['operand']) + prologue_signals.append(self.makePrologueRoutingCrossbarPackets(operation['time_step'], routing_xbar_idx + 1)) + + # mark this time slot is not empty + has_addrs.append(ctrl_addr) + instruction_signals = InstructionSignals( id_ = self.id_, operations = instruction['operations'], opcode_in_EIR = instruction['operations'][0]['opcode'], # transient TODO: make it general - ctrl_addr = instruction['timestep'] % self.ii, + ctrl_addr = ctrl_addr, IntraCgraPktType = self.IntraCgraPktType, CgraPayloadType = self.CgraPayloadType, TileInType = self.TileInType, @@ -586,42 +960,85 @@ def makeTileSignals(self): B1Type = self.B1Type, B2Type = self.B2Type, RegIdxType = self.RegIdxType, - CtrlAddrType = self.CtrlAddrType) + CtrlAddrType = self.CtrlAddrType, + accumulate_add_to_src_reg = self.accumulate_add_to_src_reg) all_instruction_signals.append(instruction_signals) - + const = instruction_signals.buildCtrlPkt() if const is not None: consts.extend(const) - + + print(f"[Core {self.id_}] All {len(instructions)} instructions processed, building const signals...") + sys.stdout.flush() + # make the const signals - for idx, const_operand in enumerate(consts): - const_pkt = self.IntraCgraPktType(0, self.id_, + #print("\n\n\n\n\n\n\n\n\n\n\n\n\n") + # ConstQueueDynamicRTL consumes constants in controller execution order + # (the order CONFIG packets are generated), not in scheduled time_step + # order. Some schedules have non-monotonic time_steps across + # index_per_ii slots, so sorting here misaligns const-using ops. + #print(consts) + for idx, (const_operand, _) in enumerate(consts): + operand_str = const_operand['operand'] + if operand_str.startswith('#'): + const_val = int(operand_str[1:]) + elif operand_str.startswith('arg'): + # Function argument: look up in arg_map + const_val = self.arg_map.get(operand_str, 0) + else: + const_val = int(operand_str) + const_pkt = self.IntraCgraPktType(0, self.id_, payload = self.CgraPayloadType(self.CMD_CONST_, - data = self.DataType(int(const_operand['operand'][1:] if const_operand['operand'].startswith('#') else const_operand['operand']), 1))) + data = self.DataType(const_val, 1))) all_signals.append(const_pkt) - + + # If any instruction uses 2D GEP, send stride configuration packet + if self.gep_stride is not None: + has_gep_2d = False + for instruction in instructions: + for operation in instruction['operations']: + if operation['opcode'] == 'GEP' and _is_take_up_fu_operation(operation): + try: + src_operands = operation['src_operands'] + except: + src_operands = [] + non_imm_count = sum(1 for op in src_operands if _type(op) != 'IMM') + if non_imm_count >= 2: + has_gep_2d = True + break + if has_gep_2d: + break + if has_gep_2d: + from lib.cmd_type import CMD_CONFIG_GEP_STRIDE + gep_stride_pkt = self.IntraCgraPktType(0, self.id_, + payload = self.CgraPayloadType(CMD_CONFIG_GEP_STRIDE, + data = self.DataType(self.gep_stride, 1))) + all_signals.append(gep_stride_pkt) + + print(f"[Core {self.id_}] Building pre-config and main packets...") + sys.stdout.flush() + # make the pre-configuration - ii_pkt = self.IntraCgraPktType(0, self.id_, + ii_pkt = self.IntraCgraPktType(0, self.id_, payload = self.CgraPayloadType(self.CMD_CONFIG_COUNT_PER_ITER_, data = self.DataType(self.ii, 1))) all_signals.append(ii_pkt) - loop_times_pkt = self.IntraCgraPktType(0, self.id_, + loop_times_pkt = self.IntraCgraPktType(0, self.id_, payload = self.CgraPayloadType(self.CMD_CONFIG_TOTAL_CTRL_COUNT_, data = self.DataType(self.loop_times, 1))) all_signals.append(loop_times_pkt) - - - main_signals = [] + + + main_signals = {} # make the main packets for instruction_signals in all_instruction_signals: pkt = instruction_signals.makeCtrlPkt() - main_signals.append(pkt) - - + main_signals[int(pkt.payload.ctrl_addr)] = pkt + + # fill the non-existent timesteps with NAH packets filled_main_signals = [] - idx = 0 for timestep in range(self.ii): if timestep not in has_addrs: NAH_SIGNAL = self.IntraCgraPktType(0, self.id_, @@ -629,17 +1046,16 @@ def makeTileSignals(self): ctrl = self.CtrlType(OPT_NAH, [self.FuInType(i) for i in range(1, 5)], [self.TileInType(0)] * 8, - [self.FuOutType(0)] * 8))) + [self.FuOutType(0)] * 8))) filled_main_signals.append(NAH_SIGNAL) else: - filled_main_signals.append(main_signals[idx]) # WARN: generated signals must be in order - idx += 1 - - + filled_main_signals.append(main_signals[timestep]) + + all_signals.extend(filled_main_signals) - + # make prologue packets - # re-order the prologue packets + # re-order the prologue packets ''' ordered_prologue_signals = [] for pkt in prologue_signals: @@ -651,29 +1067,32 @@ def makeTileSignals(self): for pkt in prologue_signals: if pkt.cmd == self.CMD_CONFIG_PROLOGUE_FU_CROSSBAR_: ordered_prologue_signals.append(pkt) - + all_signals.extend(ordered_prologue_signals) ''' all_signals.extend(prologue_signals) # make the launch packet - launch_pkt = self.IntraCgraPktType(0, self.id_, + launch_pkt = self.IntraCgraPktType(0, self.id_, payload = self.CgraPayloadType(self.CMD_LAUNCH_)) all_signals.append(launch_pkt) - + + print(f"[Core {self.id_}] makeTileSignals() DONE, {len(all_signals)} total signals") + sys.stdout.flush() + return all_signals class ScriptFactory: FromFu = 0 FromRouting = 1 - - def __init__(self, - path, - CtrlType, - IntraCgraPktType, - CgraPayloadType, - TileInType, - FuOutType, - CMD_CONFIG_input, - FuInType, + + def __init__(self, + path, + CtrlType, + IntraCgraPktType, + CgraPayloadType, + TileInType, + FuOutType, + CMD_CONFIG_input, + FuInType, ii, loop_times, CMD_CONST_input, @@ -689,11 +1108,22 @@ def __init__(self, RegIdxType, CtrlAddrType, DataAddrType, - num_registers_per_reg_bank=None): + num_registers_per_reg_bank=None, + arg_map=None, + gep_stride=None, + use_time_step_ctrl_addr=False, + accumulate_add_to_src_reg=False): # Allow overriding the default register cluster size. global REG_CLUSTER_SIZE if num_registers_per_reg_bank is not None: REG_CLUSTER_SIZE = int(num_registers_per_reg_bank) + # arg_map: dict mapping function argument names (e.g. "arg6") to + # integer values (e.g. base addresses). Used for GEP operations + # that reference function parameters. + self.arg_map = arg_map if arg_map is not None else {} + self.gep_stride = gep_stride # stride for 2D GEP operations + self.use_time_step_ctrl_addr = use_time_step_ctrl_addr + self.accumulate_add_to_src_reg = accumulate_add_to_src_reg self.yaml_struct = yaml.load(open(path, 'r'), Loader=yaml.FullLoader) self.path = path self.CtrlType = CtrlType @@ -718,19 +1148,23 @@ def __init__(self, self.RegIdxType = RegIdxType self.CtrlAddrType = CtrlAddrType self.DataAddrType = DataAddrType - + def makeVectorCGRAPkts(self): - + pkts = {} cores = self.yaml_struct['array_config']['cores'] - - + + for core in cores: x, y = core['column'], core['row'] entry = core['entries'][0] instructions = entry['instructions'] id_ = core['core_id'] - + + import sys + print(f"\n>>> makeVectorCGRAPkts: Starting Core {id_} at ({x},{y}) with {len(instructions)} instructions") + sys.stdout.flush() + tile_signals = TileSignals( CtrlType = self.CtrlType, IntraCgraPktType = self.IntraCgraPktType, @@ -740,8 +1174,8 @@ def makeVectorCGRAPkts(self): CMD_CONFIG_input = self.CMD_CONFIG_, FuInType = self.FuInType, id_ = id_, - loop_times = self.loop_times, - ii = self.ii, + loop_times = self.loop_times, + ii = self.ii, instructions = instructions, CMD_CONST_input = self.CMD_CONST_, CMD_CONFIG_COUNT_PER_ITER_input = self.CMD_CONFIG_COUNT_PER_ITER_, @@ -756,19 +1190,27 @@ def makeVectorCGRAPkts(self): RegIdxType = self.RegIdxType, CtrlAddrType = self.CtrlAddrType, DataAddrType = self.DataAddrType, + arg_map = self.arg_map, + gep_stride = self.gep_stride, + use_time_step_ctrl_addr = self.use_time_step_ctrl_addr, + accumulate_add_to_src_reg = self.accumulate_add_to_src_reg, ) tile_signals = tile_signals.makeTileSignals() pkts[(x, y)] = tile_signals - + print(f">>> makeVectorCGRAPkts: Core {id_} DONE") + sys.stdout.flush() + + print(">>> makeVectorCGRAPkts: ALL CORES DONE") + sys.stdout.flush() return pkts - + from validation.test.dummy import * - + if __name__ == "__main__": print("Test the Basic Functionality of the ScriptFactory") script_factory = ScriptFactory( - path = "./validation/test/fir_acceptance_test.yaml", + path = "./validation/test/gemm/gemm.yaml", CtrlType = CtrlTypeDummy, IntraCgraPktType = IntraCgraPktTypeDummy, CgraPayloadType = CgraPayloadTypeDummy, @@ -776,7 +1218,7 @@ def makeVectorCGRAPkts(self): FuOutType = FuOutTypeDummy, CMD_CONFIG_input = CMD_CONFIG_Dummy(), FuInType = FuInTypeDummy, - ii = 4, + ii = 17, loop_times = 2, CMD_CONST_input = CMD_CONST_Dummy(), CMD_CONFIG_COUNT_PER_ITER_input = CMD_CONFIG_COUNT_PER_ITER_Dummy(), @@ -792,11 +1234,10 @@ def makeVectorCGRAPkts(self): CtrlAddrType = CtrlAddrTypeDummy, DataAddrType = DataAddrTypeDummy, ) - + pkts = script_factory.makeVectorCGRAPkts() for x, y in pkts: print(f"Tile ({x}, {y}):") for pkt in pkts[(x, y)]: print(pkt) print("--------------------------------") - \ No newline at end of file diff --git a/validation/test/conv/conv_small.yaml b/validation/test/conv/conv_small.yaml new file mode 100644 index 00000000..e604eae9 --- /dev/null +++ b/validation/test/conv/conv_small.yaml @@ -0,0 +1,474 @@ +array_config: + columns: 4 + rows: 4 + compiled_ii: 5 + cores: + - column: 2 + row: 0 + core_id: "2" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 1 + operations: + - opcode: "GRANT_PREDICATE" + id: 48 + time_step: 11 + invalid_iterations: 2 + src_operands: + - operand: "NORTH" + color: "RED" + - operand: "$0" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - index_per_ii: 2 + operations: + - opcode: "DATA_MOV" + id: 20 + time_step: 7 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - opcode: "RETURN_VALUE" + id: 52 + time_step: 12 + invalid_iterations: 2 + src_operands: + - operand: "$0" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "DATA_MOV" + id: 440000 + time_step: 9 + invalid_iterations: 1 + src_operands: + - operand: "EAST" + color: "RED" + dst_operands: + - operand: "NORTH" + color: "RED" + - column: 3 + row: 0 + core_id: "3" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 1 + operations: + - opcode: "LOAD" + id: 39 + time_step: 6 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - index_per_ii: 2 + operations: + - opcode: "LOAD" + id: 40 + time_step: 7 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$8" + color: "RED" + - index_per_ii: 3 + operations: + - opcode: "MUL" + id: 43 + time_step: 8 + invalid_iterations: 1 + src_operands: + - operand: "$0" + color: "RED" + - operand: "$8" + color: "RED" + dst_operands: + - operand: "WEST" + color: "RED" + - column: 1 + row: 1 + core_id: "5" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 1 + operations: + - opcode: "DATA_MOV" + id: 28 + time_step: 6 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - opcode: "GRANT_PREDICATE" + id: 49 + time_step: 11 + invalid_iterations: 2 + src_operands: + - operand: "EAST" + color: "RED" + - operand: "$0" + color: "RED" + dst_operands: + - operand: "NORTH" + color: "RED" + - column: 2 + row: 1 + core_id: "6" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 0 + operations: + - opcode: "ADD" + id: 45 + time_step: 10 + invalid_iterations: 2 + src_operands: + - operand: "SOUTH" + color: "RED" + - operand: "$0" + color: "RED" + dst_operands: + - operand: "WEST" + color: "RED" + - operand: "SOUTH" + color: "RED" + - index_per_ii: 1 + operations: + - opcode: "DATA_MOV" + id: 200002 + time_step: 6 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "DATA_MOV" + id: 10 + time_step: 9 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - column: 3 + row: 1 + core_id: "7" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 0 + operations: + - opcode: "GEP" + id: 34 + time_step: 5 + invalid_iterations: 1 + src_operands: + - operand: "arg7" + color: "RED" + - operand: "$0" + color: "RED" + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - opcode: "DATA_MOV" + id: 30 + time_step: 5 + invalid_iterations: 1 + src_operands: + - operand: "NORTH" + color: "RED" + dst_operands: + - operand: "$8" + color: "RED" + - index_per_ii: 1 + operations: + - opcode: "GEP" + id: 35 + time_step: 6 + invalid_iterations: 1 + src_operands: + - operand: "arg6" + color: "RED" + - operand: "$0" + color: "RED" + - operand: "$8" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - index_per_ii: 3 + operations: + - opcode: "DIV" + id: 19 + time_step: 3 + invalid_iterations: 0 + src_operands: + - operand: "NORTH" + color: "RED" + - operand: "#3" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "ZEXT" + id: 26 + time_step: 4 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - column: 1 + row: 2 + core_id: "9" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 0 + operations: + - opcode: "DATA_MOV" + id: 280000 + time_step: 5 + invalid_iterations: 1 + src_operands: + - operand: "EAST" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - index_per_ii: 2 + operations: + - opcode: "DATA_MOV" + id: 5 + time_step: 2 + invalid_iterations: 0 + src_operands: + - operand: "EAST" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - opcode: "PHI_START" + id: 7 + time_step: 7 + invalid_iterations: 1 + src_operands: + - operand: "$0" + color: "RED" + - operand: "SOUTH" + color: "RED" + dst_operands: + - operand: "EAST" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "DATA_MOV" + id: 200000 + time_step: 4 + invalid_iterations: 0 + src_operands: + - operand: "EAST" + color: "RED" + dst_operands: + - operand: "EAST" + color: "RED" + - column: 2 + row: 2 + core_id: "10" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 0 + operations: + - opcode: "GRANT_PREDICATE" + id: 33 + time_step: 5 + invalid_iterations: 1 + src_operands: + - operand: "$8" + color: "RED" + - operand: "$0" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - opcode: "DATA_MOV" + id: 200001 + time_step: 5 + invalid_iterations: 1 + src_operands: + - operand: "WEST" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - index_per_ii: 1 + operations: + - opcode: "PHI_START" + id: 6 + time_step: 1 + invalid_iterations: 0 + src_operands: + - operand: "EAST" + color: "RED" + - operand: "$0" + color: "RED" + dst_operands: + - operand: "EAST" + color: "RED" + - operand: "$0" + color: "RED" + - opcode: "DATA_MOV" + id: 50000 + time_step: 1 + invalid_iterations: 0 + src_operands: + - operand: "EAST" + color: "RED" + dst_operands: + - operand: "WEST" + color: "RED" + - index_per_ii: 2 + operations: + - opcode: "ADD" + id: 11 + time_step: 2 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + - operand: "#1" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - operand: "$8" + color: "RED" + - index_per_ii: 3 + operations: + - opcode: "ICMP_EQ" + id: 17 + time_step: 3 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + - operand: "#6" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - operand: "WEST" + color: "RED" + - opcode: "DATA_MOV" + id: 100000 + time_step: 8 + invalid_iterations: 1 + src_operands: + - operand: "WEST" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "NOT" + id: 24 + time_step: 4 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + dst_operands: + - operand: "WEST" + color: "RED" + - operand: "$0" + color: "RED" + - column: 3 + row: 2 + core_id: "11" + entries: + - entry_id: "entry0" + instructions: + - index_per_ii: 0 + operations: + - opcode: "GRANT_ONCE" + id: 0 + time_step: 0 + invalid_iterations: 0 + src_operands: + - operand: "#0" + color: "RED" + dst_operands: + - operand: "WEST" + color: "RED" + - index_per_ii: 2 + operations: + - opcode: "CAST_TRUNC" + id: 12 + time_step: 2 + invalid_iterations: 0 + src_operands: + - operand: "WEST" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" + - operand: "$0" + color: "RED" + - index_per_ii: 3 + operations: + - opcode: "REM" + id: 18 + time_step: 3 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + - operand: "#3" + color: "RED" + dst_operands: + - operand: "$0" + color: "RED" + - index_per_ii: 4 + operations: + - opcode: "ZEXT" + id: 25 + time_step: 4 + invalid_iterations: 0 + src_operands: + - operand: "$0" + color: "RED" + dst_operands: + - operand: "SOUTH" + color: "RED" From 555104bedf06b78ad6ffd8b65026985b03cf5e8a Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:45:53 +0800 Subject: [PATCH 05/60] Retire pending const queue consumes --- mem/const/ConstQueueDynamicRTL.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/mem/const/ConstQueueDynamicRTL.py b/mem/const/ConstQueueDynamicRTL.py index bf003e39..76db513f 100644 --- a/mem/const/ConstQueueDynamicRTL.py +++ b/mem/const/ConstQueueDynamicRTL.py @@ -34,6 +34,12 @@ def construct(s, DataType, const_mem_size): # write cursor and read cursor s.wr_cur = Wire(WrCurType) s.rd_cur = Wire(AddrType) + # Latch for a consumption whose val/rdy handshake already completed but + # whose ctrl_proceed has not yet pulsed. Without this, the queue advance + # is lost when the reader asserts rdy one cycle but ctrl_proceed only + # fires later (e.g., at an iteration boundary where some sub-module of + # the tile delays ctrl_mem.send_ctrl.rdy). + s.consume_pending = Wire(1) # Interface s.send_const = SendIfcRTL(DataType) @@ -93,14 +99,26 @@ def update_send_val(): def update_rd_cur(): if s.reset | s.clear: s.rd_cur <<= 0 + s.consume_pending <<= 0 else: - # Checks whether the "reader" successfully read the data at rd_cur, - # and proceed rd_cur accordingly. - if s.send_const.rdy & s.ctrl_proceed: - if zext((s.rd_cur), WrCurType) < (s.wr_cur - 1): + # A consumption is "owed" whenever the reader raises rdy, or one + # was already pending from an earlier cycle. It retires on the next + # ctrl_proceed pulse. (Matches the original advance condition when + # rdy and ctrl_proceed happen in the same cycle, but also covers the + # case where rdy was asserted one cycle and ctrl_proceed only pulses + # later; previously that handshake was silently lost.) + handshake_now = s.send_const.rdy + consume_retire = (s.consume_pending | handshake_now) & s.ctrl_proceed + if consume_retire: + if zext(s.rd_cur, WrCurType) < (s.wr_cur - 1): s.rd_cur <<= s.rd_cur + 1 else: s.rd_cur <<= 0 + s.consume_pending <<= 0 + else: + # Remember an in-flight handshake whose ctrl_proceed has not yet + # pulsed, so a later ctrl_proceed still advances the queue. + s.consume_pending <<= s.consume_pending | handshake_now def line_trace(s, verbosity = 0): From 5904e1a2ab8560bee41a6622f96ff2a4d300e959 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:43:35 +0800 Subject: [PATCH 06/60] Fix dynamic ctrl termination --- mem/ctrl/CtrlMemDynamicRTL.py | 40 ++++++++++++------------- mem/ctrl/test/CtrlMemDynamicRTL_test.py | 8 ++--- 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index cbb330d8..960fad1b 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -30,7 +30,6 @@ def construct(s, IntraCgraPktType, CgraPayloadType = IntraCgraPktType.get_field_type(kAttrPayload) CtrlType = CgraPayloadType.get_field_type(kAttrCtrl) - IntraPktTileIdType = IntraCgraPktType.get_field_type(kAttrSrc) # The total_ctrl_steps indicates the number of steps the ctrl # signals should proceed. For example, if the number of ctrl # signals is 4 and they need to repeat 5 times, then the total @@ -41,7 +40,7 @@ def construct(s, IntraCgraPktType, CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) PCType = mk_bits(clog2(ctrl_count_per_iter + 1)) UpperBoundType = mk_bits(clog2(ctrl_mem_size + 1)) - TimeType = mk_bits(clog2(MAX_CTRL_COUNT + 1)) + TimeType = mk_bits(clog2(max(MAX_CTRL_COUNT, total_ctrl_steps) + 1)) PrologueCountType = mk_bits(clog2(PROLOGUE_MAX_COUNT + 1)) num_routing_xbar_inports = num_tile_inports + num_fu_inports TileInPortType = mk_bits(clog2(num_routing_xbar_inports)) @@ -135,8 +134,10 @@ def update_msg(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_LOWER) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_UPDATE_COUNTER_SHADOW_VALUE) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER)): + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER) | \ + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE)): s.send_to_element.msg @= s.recv_pkt_from_controller_queue.send.msg.payload s.send_to_element.val @= 1 @@ -161,8 +162,10 @@ def update_msg(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_LOWER) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_UPDATE_COUNTER_SHADOW_VALUE) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER): + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER) | \ + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE): s.recv_pkt_from_controller_queue.send.rdy @= 1 # TODO: Extend for the other commands. Maybe another queue to # handle complicated actions. @@ -180,20 +183,13 @@ def update_send_pkt_to_controller(): if s.start_iterate_ctrl == b1(1): if s.recv_from_element_queue.send.val & (~s.sent_complete): s.send_pkt_to_controller.msg @= \ - IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, + IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, s.recv_from_element_queue.send.msg) s.send_pkt_to_controller.val @= 1 s.recv_from_element_queue.send.rdy @= s.send_pkt_to_controller.rdy - elif ((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ - (s.reg_file.rdata[0].operation == OPT_START): - # Sends COMPLETE signal to Controller when the last ctrl signal is done. - if ~s.sent_complete & (s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val) & s.start_iterate_ctrl: - s.send_pkt_to_controller.msg @= \ - IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) - s.send_pkt_to_controller.val @= 1 @update - def update_send_ctrl_val(): + def update_send_ctrl(): s.send_ctrl.val @= 0 if s.start_iterate_ctrl == b1(1): if s.sent_complete: @@ -207,8 +203,6 @@ def update_send_ctrl_val(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_TERMINATE): s.send_ctrl.val @= b1(0) - @update - def update_send_ctrl_msg(): for i in range(num_fu_inports): s.send_ctrl.msg.fu_in[i] @= s.reg_file.rdata[0].fu_in[i] s.send_ctrl.msg.write_reg_from[i] @= s.reg_file.rdata[0].write_reg_from[i] @@ -219,11 +213,16 @@ def update_send_ctrl_msg(): s.send_ctrl.msg.routing_xbar_outport[i] @= s.reg_file.rdata[0].routing_xbar_outport[i] s.send_ctrl.msg.fu_xbar_outport[i] @= s.reg_file.rdata[0].fu_xbar_outport[i] s.send_ctrl.msg.vector_factor_power @= s.reg_file.rdata[0].vector_factor_power - s.send_ctrl.msg.is_last_ctrl @= s.reg_file.rdata[0].is_last_ctrl - # Sets each FU's op code as NAH when prologue execution has not completed. - # As FU is supposed to do nothing during prologue. - if s.prologue_count_outport_fu != 0: - s.send_ctrl.msg.operation @= OPT_NAH + s.send_ctrl.msg.is_last_ctrl @= \ + s.reg_file.rdata[0].is_last_ctrl | \ + ((s.total_ctrl_steps_val > 0) & + (s.times == s.total_ctrl_steps_val - TimeType(1))) + # Keep downstream datapath blocks inactive unless this cycle is + # actually issuing a control word. FU prologue is handled in + # FlexibleFuRTL so the real control word remains visible to the + # crossbars and debug signals. + if ~s.send_ctrl.val: + s.send_ctrl.msg.operation @= OPT_START else: s.send_ctrl.msg.operation @= s.reg_file.rdata[0].operation @@ -345,4 +344,3 @@ def update_total_ctrl_steps(): def line_trace(s): config_mem_str = "|".join([str(data) for data in s.reg_file.regs]) return f'reg_file.raddr[0]: {s.reg_file.raddr[0]} || sent_complete: {s.sent_complete} || times: {s.times} || total_ctrl_steps_val: {s.total_ctrl_steps_val} || start_iterate_ctrl: {s.start_iterate_ctrl}|| recv_pkt: {s.recv_pkt_from_controller.msg}.recv_rdy:{s.recv_pkt_from_controller.rdy} || control signal content: [{config_mem_str}] || ctrl_out: {s.send_ctrl.msg}, send_ctrl.val: {s.send_ctrl.val}, send_ctrl.rdy: {s.send_ctrl.rdy}, send_pkt.msg.payload.cmd: {s.send_pkt_to_controller.msg.payload.cmd}, send_pkt.val: {s.send_pkt_to_controller.val}, ctrl_count_per_iter_val: {s.ctrl_count_per_iter_val}, ctrl_count_lower_bound: {s.ctrl_count_lower_bound}' - diff --git a/mem/ctrl/test/CtrlMemDynamicRTL_test.py b/mem/ctrl/test/CtrlMemDynamicRTL_test.py index 417e7a08..178608c6 100644 --- a/mem/ctrl/test/CtrlMemDynamicRTL_test.py +++ b/mem/ctrl/test/CtrlMemDynamicRTL_test.py @@ -131,8 +131,8 @@ def test_ctrl(): IntraCgraPktType(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_LAUNCH, ctrl = CtrlType(OPT_NAH, pick_register), ctrl_addr = 0))] sink_out = [DataType(7, 1), DataType(4, 1), DataType(5, 1), DataType(9, 1)] - complete_signal_sink_out = [ - IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE))] + # Completion must come from an explicit RET, not control-memory timeout. + complete_signal_sink_out = [] ctrl_count_per_iter = len(src_ctrl_pkt) - 1 total_ctrl_steps_val = len(src_ctrl_pkt) - 1 @@ -221,8 +221,8 @@ def test_ctrl_bound(): # As ctrl signals with ctrl_addr = 1 and ctrl_addr = 2 are both OPT_SUB, # The outputs are 1-6 5-1 sink_out = [DataType(-5, 1), DataType(4, 1)] - complete_signal_sink_out = [ - IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE))] + # Completion must come from an explicit RET, not control-memory timeout. + complete_signal_sink_out = [] th = TestHarness(MemUnit, IntraCgraPktType, From de4427c9c89d6656ae31b2146b2f360ad681ea16 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:43:47 +0800 Subject: [PATCH 07/60] Bypass final RET routing write --- mem/register_cluster/RegisterClusterRTL.py | 42 +++++++++++++++---- .../test/RegisterClusterRTL_test.py | 32 +++++++++++++- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index f6a7ad25..cf7bb043 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -39,6 +39,8 @@ def construct(s, DataType, CtrlType, num_reg_banks, s.recv_data_from_routing_crossbar = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] s.recv_data_from_fu_crossbar = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] s.recv_data_from_const = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] + s.write_data_from_routing_crossbar = [InPort(DataType) for _ in range(num_reg_banks)] + s.write_valid_from_routing_crossbar = [InPort(b1) for _ in range(num_reg_banks)] s.send_data_to_fu = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] # Direct output from register banks towards routing crossbar (bypasses FU). s.send_data_to_routing_crossbar = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] @@ -50,10 +52,10 @@ def construct(s, DataType, CtrlType, num_reg_banks, # Connections. for i in range(num_reg_banks): s.reg_bank[i].inport_opt //= s.inport_opt - s.reg_bank[i].inport_wdata[PORT_INDEX_ROUTING_CROSSBAR] //= s.recv_data_from_routing_crossbar[i].msg + s.reg_bank[i].inport_wdata[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_data_from_routing_crossbar[i] s.reg_bank[i].inport_wdata[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].msg s.reg_bank[i].inport_wdata[PORT_INDEX_CONST] //= s.recv_data_from_const[i].msg - s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.recv_data_from_routing_crossbar[i].val + s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_valid_from_routing_crossbar[i] s.reg_bank[i].inport_valid[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].val s.reg_bank[i].inport_valid[PORT_INDEX_CONST] //= s.recv_data_from_const[i].val @@ -70,28 +72,50 @@ def update_msgs_signals(): s.send_data_to_routing_crossbar[i].val @= 0 for i in range(num_reg_banks): + active_ctrl = s.inport_opt.operation != OPT_START read_towards = s.inport_opt.read_reg_towards[i] # Checks if data should go towards FU (1 or 3) - reg_towards_fu = (read_towards == kReadTowardsFu) | (read_towards == kReadTowardsBoth) + reg_towards_fu = active_ctrl & \ + ((read_towards == kReadTowardsFu) | (read_towards == kReadTowardsBoth)) # Checks if data should go towards routing_xbar (2 or 3) - reg_towards_routing_xbar = (read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth) + reg_towards_routing_xbar = active_ctrl & \ + ((read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth)) + ret_last_routing_write_bypass = active_ctrl & \ + s.inport_opt.is_last_ctrl & \ + (s.inport_opt.operation == OPT_RET) & \ + reg_towards_fu & \ + (s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + (s.inport_opt.write_reg_idx[i] == s.inport_opt.read_reg_idx[i]) & \ + s.write_valid_from_routing_crossbar[i] # Data from register bank has priority over routing crossbar data for FU path. # Note: reg_bank[i].send_data.val is set based on read_reg_towards in RegisterBankRTL. - if s.reg_bank[i].send_data.val & reg_towards_fu: + if ret_last_routing_write_bypass: + s.send_data_to_fu[i].msg @= \ + s.write_data_from_routing_crossbar[i] + elif s.reg_bank[i].send_data.val & reg_towards_fu: s.send_data_to_fu[i].msg @= \ s.reg_bank[i].send_data.msg elif s.recv_data_from_routing_crossbar[i].val: s.send_data_to_fu[i].msg @= \ s.recv_data_from_routing_crossbar[i].msg - s.send_data_to_fu[i].val @= \ - s.recv_data_from_routing_crossbar[i].val | \ + s.send_data_to_fu[i].val @= active_ctrl & \ + (ret_last_routing_write_bypass | \ + s.recv_data_from_routing_crossbar[i].val) | \ (s.reg_bank[i].send_data.val & reg_towards_fu) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy - s.recv_data_from_routing_crossbar[i].rdy @= ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) \ - & (s.inport_opt.operation == OPT_NAH)) | s.send_data_to_fu[i].rdy + # Avoid artificial cycle gaps when the FU is reading from the local + # register bank or not consuming this FU input. In those cases the + # routing path should not hold the upstream sender back. + s.recv_data_from_routing_crossbar[i].rdy @= \ + (~active_ctrl | \ + ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + (s.inport_opt.operation == OPT_NAH)) | \ + (s.inport_opt.fu_in[i] == 0) | \ + reg_towards_fu | \ + s.send_data_to_fu[i].rdy) s.recv_data_from_fu_crossbar[i].rdy @= 1 s.recv_data_from_const[i].rdy @= 1 diff --git a/mem/register_cluster/test/RegisterClusterRTL_test.py b/mem/register_cluster/test/RegisterClusterRTL_test.py index d7fbb4a3..1d4084f2 100644 --- a/mem/register_cluster/test/RegisterClusterRTL_test.py +++ b/mem/register_cluster/test/RegisterClusterRTL_test.py @@ -45,7 +45,7 @@ class TestHarness(Component): def construct(s, DataType, ConfigType, num_reg_banks, num_registers, src_opt, src_msgs_routing_xbar, src_msgs_fu_xbar, - src_msgs_const, sink_msgs): + src_msgs_const, sink_msgs, sink_initial_delay = 0): s.num_reg_banks = num_reg_banks s.src_opt = Wire(ConfigType) @@ -57,7 +57,8 @@ def construct(s, DataType, ConfigType, num_reg_banks, num_registers, s.src_const = [TestSrcRTL(DataType, src_msgs_const[i]) for i in range(num_reg_banks)] - s.sink = [TestSinkRTL(DataType, sink_msgs[i]) + s.sink = [TestSinkRTL(DataType, sink_msgs[i], + initial_delay = sink_initial_delay) for i in range(num_reg_banks)] s.reg_cluster = RegisterClusterRTL(DataType, ConfigType, num_reg_banks, @@ -172,6 +173,33 @@ def test_reg_bank(): expected_sink_data) run_sim(th) +# Concrete example: the last RET control writes routing token 42 into +# register 3 and reads register 3 for the FU in the same cycle. Without this +# bypass the FU sees the old register value, so the returned value is stale. +def test_last_ret_bypasses_same_cycle_routing_write(): + DataType = mk_data(16, 1) + num_reg_banks = 4 + num_registers = 16 + ConfigType = mk_ctrl(4, 2, 4, 4, num_registers) + FuInType = mk_bits(clog2(5)) + + ctrl = ConfigType(OPT_RET, [FuInType(x + 1) for x in range(4)]) + ctrl.is_last_ctrl = 1 + ctrl.write_reg_from[0] = b2(PORT_ROUTING_CROSSBAR) + ctrl.write_reg_idx[0] = b4(3) + ctrl.read_reg_towards[0] = b2(READ_TOWARDS_FU) + ctrl.read_reg_idx[0] = b4(3) + + th = TestHarness( + DataType, ConfigType, num_reg_banks, num_registers, ctrl, + [[DataType(42, 1)], [], [], []], + [[] for _ in range(num_reg_banks)], + [[] for _ in range(num_reg_banks)], + [[DataType(42, 1)], [], [], []], + sink_initial_delay = 1, + ) + run_sim(th) + #------------------------------------------------------------------------- # Extended test harness that also sinks send_data_to_routing_crossbar #------------------------------------------------------------------------- From 74013d22e93838f7058d08c3ca3c5e547dd31ca2 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:43:45 +0800 Subject: [PATCH 08/60] Fix Crossbar prologue token alignment --- noc/CrossbarRTL.py | 87 +++++++++++++++++++++++++----------- noc/test/CrossbarRTL_test.py | 60 ++++++++++++++++++++++++- 2 files changed, 120 insertions(+), 27 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index 3a6fc145..2b56d500 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -58,6 +58,7 @@ def construct(s, s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) s.crossbar_id = InPort(b1) s.compute_done = InPort(b1) + s.drain_when_inactive = InPort(b1) s.ctrl_addr_inport = InPort(CtrlAddrType) @@ -89,8 +90,8 @@ def construct(s, # Prologue-related wires and registers, which are used to indicate # whether the prologue steps have already been satisfied. - s.during_prologue_allowing_vector = Wire(num_outports) - s.recv_valid_or_during_prologue_allowing_vector = Wire(num_outports) + s.prologue_allowing_vector = Wire(num_outports) + s.recv_valid_or_prologue_allowing_vector = Wire(num_outports) s.prologue_counter = [[Wire(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] s.prologue_counter_next = [[Wire(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] s.prologue_count_inport = [[InPort(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] @@ -125,7 +126,9 @@ def update_signal(): s.all_send_accepted @= 0 for i in range(num_inports): - s.recv_data[i].rdy @= reduce_and(s.recv_valid_vector) & \ + # Prologued inputs are still required so the skipped token is + # consumed; send_required_vector decides whether it is forwarded. + s.recv_data[i].rdy @= reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ s.all_send_accepted & \ s.recv_required_vector[i] @@ -134,17 +137,32 @@ def update_signal(): # yet accepted in a previous cycle. This prevents duplicate # delivery without making val depend on rdy (send_accepted is # a register, so this is purely val-depends-on-registered-state). - s.send_data[i].val @= reduce_and(s.recv_valid_vector) & \ + # + # Uses recv_valid_or_prologue_allowing_vector as the gate so + # that prologued inputs (bypassed) do not suppress sends on + # non-prologued outputs. send_required_vector already + # excludes prologued outputs, so they will never assert val. + s.send_data[i].val @= reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ s.send_required_vector[i] & \ ~s.send_accepted[i] - if reduce_and(s.recv_valid_vector) & \ + if reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ s.send_required_vector[i] & \ ~s.send_accepted[i]: s.send_data[i].msg.payload @= s.recv_data_msg[s.in_dir_local[i]].payload s.send_data[i].msg.predicate @= s.recv_data_msg[s.in_dir_local[i]].predicate s.recv_opt.rdy @= s.all_send_accepted & \ - reduce_and(s.recv_valid_or_during_prologue_allowing_vector) + reduce_and(s.recv_valid_or_prologue_allowing_vector) + else: + # Unconfigured / idle tiles must not backpressure the fabric. + # If traffic reaches a crossbar with no active route, treat it + # as a sink so stray messages can drain instead of deadlocking + # upstream producers. When this crossbar has already completed the + # current ctrl step, keep inputs backpressured so next-step tokens + # are not dropped while the tile waits for ctrl_proceed. + if s.drain_when_inactive: + for i in range(num_inports): + s.recv_data[i].rdy @= 1 @update_ff def update_prologue_counter(): @@ -199,22 +217,22 @@ def update_send_accepted_next(): @update def update_prologue_allowing_vector(): - s.during_prologue_allowing_vector @= 0 + s.prologue_allowing_vector @= 0 for i in range(num_outports): if s.in_dir[i] > 0: # Records whether the prologue steps have already been satisfied. - s.during_prologue_allowing_vector[i] @= \ - (s.prologue_counter[s.ctrl_addr_inport][s.in_dir_local[i]] < \ - s.prologue_count_wire[s.ctrl_addr_inport][s.in_dir_local[i]]) + s.prologue_allowing_vector[i] @= \ + s.prologue_counter[s.ctrl_addr_inport][s.in_dir_local[i]] < \ + s.prologue_count_wire[s.ctrl_addr_inport][s.in_dir_local[i]] else: - s.during_prologue_allowing_vector[i] @= 0 + s.prologue_allowing_vector[i] @= 1 @update def update_prologue_or_valid_vector(): - s.recv_valid_or_during_prologue_allowing_vector @= 0 + s.recv_valid_or_prologue_allowing_vector @= 0 for i in range(num_outports): - s.recv_valid_or_during_prologue_allowing_vector[i] @= \ - s.recv_valid_vector[i] | s.during_prologue_allowing_vector[i] + s.recv_valid_or_prologue_allowing_vector[i] @= \ + s.recv_valid_vector[i] | s.prologue_allowing_vector[i] @update def update_in_dir_vector(): @@ -232,14 +250,17 @@ def update_in_dir_vector(): def update_rdy_vector(): s.send_rdy_vector @= 0 for i in range(num_outports): - # The `outport_towards_local_base_id` indicates the number of outports that go to other tiles. + # The `num_inports` indicates the number of outports that go to other tiles. # Specifically, if the compute already done, we shouldn't care the ones - # (i.e., i >= outport_towards_local_base_id) go to the FU's inports. In other words, we skip + # (i.e., i >= num_inports) go to the FU's inports. In other words, we skip # the rdy checking on the FU's inports (connecting from crossbar_outport) if # the compute is already completed. - if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id)): - s.send_rdy_vector[i] @= s.send_data[i].rdy + if s.in_dir[i] > 0: + # When prologue is active for this output's input, don't + # require the downstream channel to be ready -- we won't be + # sending any data through it during prologue anyway. + s.send_rdy_vector[i] @= s.send_data[i].rdy | \ + s.prologue_allowing_vector[i] else: s.send_rdy_vector[i] @= 1 @@ -247,8 +268,19 @@ def update_rdy_vector(): def update_valid_vector(): s.recv_valid_vector @= 0 for i in range(num_outports): - if s.in_dir[i] > 0: - s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] + if (s.in_dir[i] > 0) & \ + (~s.compute_done | (i < outport_towards_local_base_id)): + # When prologue is active for this output's input, treat the + # input as not-valid even if data is physically present in the + # channel. This prevents the crossbar from asserting + # send_data.val (and therefore potentially blocking on a full + # downstream channel) during prologue steps. Without this + # masking, stale data that happens to sit in a channel can + # trick the crossbar into attempting a send, and if the + # destination channel is full the all-or-nothing semantics + # cause a deadlock. + s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] & \ + ~s.prologue_allowing_vector[i] else: s.recv_valid_vector[i] @= 1 @@ -258,9 +290,10 @@ def update_recv_required_vector(): s.recv_required_vector[i] @= 0 for i in range(num_outports): + # Prologued inputs are still required so the skipped token is + # consumed; send_required_vector decides whether it is forwarded. if s.in_dir[i] > 0: - # Avoids crossbar mistakenly consume data during prologue. - s.recv_required_vector[s.in_dir_local[i]] @= ~s.during_prologue_allowing_vector[i] + s.recv_required_vector[s.in_dir_local[i]] @= 1 @update def update_send_required_vector(): @@ -269,7 +302,12 @@ def update_send_required_vector(): s.send_required_vector[i] @= 0 for i in range(num_outports): - if s.in_dir[i] > 0: + # An output is required only when it has a mapped input AND + # that input is NOT being bypassed by the prologue mechanism. + # During prologue, the crossbar must not attempt to send data + # on outputs fed by prologued inputs; otherwise a full + # downstream channel would block the entire crossbar. + if (s.in_dir[i] > 0) & ~s.prologue_allowing_vector[i]: s.send_required_vector[i] @= 1 @@ -278,4 +316,3 @@ def line_trace(s): recv_str = "|".join([str(x.msg) for x in s.recv_data]) out_str = "|".join([str(x.msg) for x in s.send_data]) return f"{recv_str} [{s.recv_opt.msg}] {out_str}" - diff --git a/noc/test/CrossbarRTL_test.py b/noc/test/CrossbarRTL_test.py index d730f9a5..1b96ecd8 100644 --- a/noc/test/CrossbarRTL_test.py +++ b/noc/test/CrossbarRTL_test.py @@ -24,7 +24,9 @@ class TestHarness(Component): def construct(s, CrossbarUnit, DataType, CtrlType, num_inports, num_outports, src_data, src_routing, - sink_out): + sink_out, prologue_counts = None, + drain_when_inactive = 0, + ctrl_addr_sequence = None): num_tiles = 1 ctrl_mem_size = 6 @@ -33,6 +35,7 @@ def construct(s, CrossbarUnit, DataType, CtrlType, # InType for the crossbar's crossbar_outport: clog2(num_inports+1) bits. InType = mk_bits(clog2(num_inports + 1)) + CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) s.src_opt = TestSrcRTL(CtrlType, src_routing) s.src_data = [TestSrcRTL(DataType, src_data[i]) @@ -46,8 +49,21 @@ def construct(s, CrossbarUnit, DataType, CtrlType, for i in range(num_inports): s.src_data[i].send //= s.dut.recv_data[i] for addr in range(ctrl_mem_size): - s.dut.prologue_count_inport[addr][i] //= 0 + count = 0 + if prologue_counts is not None: + count = prologue_counts.get((addr, i), 0) + s.dut.prologue_count_inport[addr][i] //= count s.src_opt.send //= s.dut.recv_opt + s.dut.compute_done //= 0 + s.dut.drain_when_inactive //= drain_when_inactive + s.ctrl_addr_sequence = ctrl_addr_sequence or [0] + + @update + def connect_ctrl_addr(): + idx = s.src_opt.idx + if idx >= len(s.ctrl_addr_sequence): + idx = len(s.ctrl_addr_sequence) - 1 + s.dut.ctrl_addr_inport @= CtrlAddrType(s.ctrl_addr_sequence[idx]) # routing_xbar_outport in CtrlType may be wider than the crossbar's InType # (because mk_ctrl now uses clog2(num_tile_inports+num_fu_inports+1)). @@ -144,3 +160,43 @@ def test_multi_cast(): num_routing_outports, src_data, src_opt, sink_out) run_sim(th) +def test_prologue_consumes_skipped_input(): + src_opt = [CtrlType(OPT_ADD, pickRegister, + [TileInType(1), TileInType(0), TileInType(0)], + [FuOutType(0), FuOutType(0), FuOutType(0)]), + CtrlType(OPT_ADD, pickRegister, + [TileInType(1), TileInType(0), TileInType(0)], + [FuOutType(0), FuOutType(0), FuOutType(0)])] + src_data = [[DataType(7, 1), DataType(9, 1)], [], []] + sink_out = [[DataType(9, 1)], [], []] + th = TestHarness(FU, DataType, CtrlType, num_tile_inports, + num_routing_outports, src_data, src_opt, sink_out, + prologue_counts = {(0, 0): 1}) + run_sim(th) + +def test_prologue_prevents_loop_token_from_terminal_route(): + src_opt = [CtrlType(OPT_ADD, pickRegister, + [TileInType(1), TileInType(0), TileInType(0)], + [FuOutType(0), FuOutType(0), FuOutType(0)]), + CtrlType(OPT_ADD, pickRegister, + [TileInType(0), TileInType(1), TileInType(0)], + [FuOutType(0), FuOutType(0), FuOutType(0)]), + CtrlType(OPT_ADD, pickRegister, + [TileInType(1), TileInType(0), TileInType(0)], + [FuOutType(0), FuOutType(0), FuOutType(0)])] + src_data = [[DataType(0, 1), DataType(1, 1), DataType(0, 1)], [], []] + sink_out = [[DataType(0, 1), DataType(0, 1)], [], []] + th = TestHarness(FU, DataType, CtrlType, num_tile_inports, + num_routing_outports, src_data, src_opt, sink_out, + prologue_counts = {(0, 0): 1}, + ctrl_addr_sequence = [4, 0, 4]) + run_sim(th) + +def test_inactive_drain_consumes_input(): + src_opt = [] + src_data = [[DataType(11, 1)], [], []] + sink_out = [[], [], []] + th = TestHarness(FU, DataType, CtrlType, num_tile_inports, + num_routing_outports, src_data, src_opt, sink_out, + drain_when_inactive = 1) + run_sim(th) From 1d89097cae690dfe2103f2d8f8ef1c59a7c07913 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 25 Jun 2026 22:46:53 +0800 Subject: [PATCH 09/60] Hold tile control until datapath completion --- tile/TileRTL.py | 90 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 0facddcf..27f0c4ba 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -39,7 +39,7 @@ class TileRTL(Component): def construct(s, IntraCgraPktType, ctrl_mem_size, data_mem_size, num_ctrl, - total_steps, num_fu_inports, num_fu_outports, + total_steps, num_fu_inports, num_fu_outports, num_tile_inports, num_tile_outports, num_cgras, num_tiles, num_registers_per_reg_bank = 16, Fu = FlexibleFuRTL, @@ -50,7 +50,9 @@ def construct(s, IntraCgraPktType, CtrlPktType = IntraCgraPktType DataType = CgraPayloadType.get_field_type(kAttrData) PredicateType = DataType.get_field_type(kAttrPredicate) + PayloadType = DataType.get_field_type(kAttrPayload) CtrlSignalType = CgraPayloadType.get_field_type(kAttrCtrl) + CmdType = CgraPayloadType.get_field_type(kAttrCmd) data_bitwidth = DataType.get_field_type(kAttrPayload).nbits # Constants. @@ -62,6 +64,10 @@ def construct(s, IntraCgraPktType, CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) DataAddrType = mk_bits(clog2(data_mem_size)) + DebugTimeType = mk_bits(clog2(max(MAX_CTRL_COUNT, total_steps) + 1)) + DebugOpType = CtrlSignalType.get_field_type(kAttrOperation) + FuInType = mk_bits(clog2(num_fu_inports + 1)) + PrologueCountType = mk_bits(clog2(PROLOGUE_MAX_COUNT + 1)) # Interfaces. s.recv_data = [RecvIfcRTL(DataType) @@ -81,7 +87,7 @@ def construct(s, IntraCgraPktType, s.to_mem_wdata = SendIfcRTL(DataType) # Components. - s.element = FlexibleFuRTL(CtrlPktType, num_fu_inports, + s.element = FlexibleFuRTL(CtrlPktType, num_fu_inports, num_fu_outports, num_tiles, FuList) s.const_mem = ConstQueueDynamicRTL(DataType, ctrl_mem_size) s.routing_crossbar = CrossbarRTL(DataType, @@ -118,6 +124,8 @@ def construct(s, IntraCgraPktType, # connected to the next tiles. s.tile_in_channel = [ChannelRTL(DataType, latency = 1) for _ in range(num_tile_inports)] + s.routing_to_reg_channel = [ChannelRTL(DataType, latency = 1) + for _ in range(num_fu_inports)] # The `tile_out_or_link` would "or" the outports of the # `tile_out_channel` and the FUs. @@ -128,6 +136,8 @@ def construct(s, IntraCgraPktType, s.element_done = Wire(1) s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) + s.routing_crossbar_idle_drain = Wire(1) + s.fu_crossbar_idle_drain = Wire(1) s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -140,11 +150,12 @@ def construct(s, IntraCgraPktType, s.fu_crossbar.tile_id //= s.tile_id s.routing_crossbar.cgra_id //= s.cgra_id s.routing_crossbar.tile_id //= s.tile_id + s.routing_crossbar.drain_when_inactive //= s.routing_crossbar_idle_drain + s.fu_crossbar.drain_when_inactive //= s.fu_crossbar_idle_drain # Assigns crossbar id. s.routing_crossbar.crossbar_id //= PORT_INDEX_ROUTING_CROSSBAR s.fu_crossbar.crossbar_id //= PORT_INDEX_FU_CROSSBAR - # Constant queue. s.element.recv_const //= s.const_mem.send_const @@ -211,18 +222,33 @@ def construct(s, IntraCgraPktType, # the FUs (via `fu_crossbar`) with the outports of the # `routing_crossbar` through the corresponding channels. for i in range(num_tile_outports): - s.fu_crossbar.send_data[i] //= s.tile_out_or_link[i].recv_fu - s.routing_crossbar.send_data[i] //= s.tile_out_or_link[i].recv_xbar - s.tile_out_or_link[i].send //= s.send_data[i] + s.tile_out_or_link[i].recv_fu.msg //= s.fu_crossbar.send_data[i].msg + s.tile_out_or_link[i].recv_fu.val //= s.fu_crossbar.send_data[i].val + s.tile_out_or_link[i].recv_xbar.msg //= \ + s.routing_crossbar.send_data[i].msg + s.tile_out_or_link[i].recv_xbar.val //= \ + s.routing_crossbar.send_data[i].val + s.tile_out_or_link[i].fu_xbar_rdy //= s.fu_crossbar.recv_opt.rdy + s.send_data[i].msg //= s.tile_out_or_link[i].send.msg + s.send_data[i].val //= s.tile_out_or_link[i].send.val + s.tile_out_or_link[i].send.rdy //= s.send_data[i].rdy # Crossbars outputs are integrated with the "register_cluster". # Whether the required operands for FU are from the "routing_crossbar" # or from the "register_cluster" depends on the control signals. for i in range(num_fu_inports): - s.routing_crossbar.send_data[num_tile_outports + i] //= \ - s.register_cluster.recv_data_from_routing_crossbar[i] - s.fu_crossbar.send_data[num_tile_outports + i] //= \ - s.register_cluster.recv_data_from_fu_crossbar[i] + s.routing_to_reg_channel[i].recv.msg //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.recv_data_from_routing_crossbar[i].msg //= \ + s.routing_to_reg_channel[i].send.msg + s.register_cluster.recv_data_from_routing_crossbar[i].val //= \ + s.routing_to_reg_channel[i].send.val + s.register_cluster.recv_data_from_fu_crossbar[i].msg //= \ + s.fu_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.recv_data_from_fu_crossbar[i].val //= \ + s.fu_crossbar.send_data[num_tile_outports + i].val s.register_cluster.recv_data_from_const[i].msg //= DataType() s.register_cluster.recv_data_from_const[i].val //= 0 @@ -231,6 +257,33 @@ def construct(s, IntraCgraPktType, s.element.recv_in[i] s.register_cluster.inport_opt //= s.ctrl_mem.send_ctrl.msg + @update + def update_routing_to_reg_inputs(): + for i in range(num_tile_outports): + s.routing_crossbar.send_data[i].rdy @= s.send_data[i].rdy + for i in range(num_fu_inports): + is_reg_write = \ + s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR + + s.routing_to_reg_channel[i].recv.val @= \ + s.routing_crossbar.send_data[num_tile_outports + i].val & \ + ~is_reg_write + s.register_cluster.write_valid_from_routing_crossbar[i] @= \ + s.routing_crossbar.send_data[num_tile_outports + i].val & \ + is_reg_write + s.routing_crossbar.send_data[num_tile_outports + i].rdy @= \ + is_reg_write | s.routing_to_reg_channel[i].recv.rdy + + @update + def update_reg_cluster_input_rdy(): + for i in range(num_tile_outports): + s.fu_crossbar.send_data[i].rdy @= s.send_data[i].rdy + for i in range(num_fu_inports): + s.routing_to_reg_channel[i].send.rdy @= \ + s.register_cluster.recv_data_from_routing_crossbar[i].rdy + s.fu_crossbar.send_data[num_tile_outports + i].rdy @= \ + s.register_cluster.recv_data_from_fu_crossbar[i].rdy + # Clear ports are only useful during context switching. # We connect to 0 to make sure they have drivers. for i in range(len(FuList)): @@ -258,7 +311,8 @@ def feed_pkt(): (s.recv_from_controller_pkt.msg.payload.cmd == CMD_LAUNCH) | \ (s.recv_from_controller_pkt.msg.payload.cmd == CMD_CONFIG_LOOP_LOWER) | \ (s.recv_from_controller_pkt.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \ - (s.recv_from_controller_pkt.msg.payload.cmd == CMD_CONFIG_LOOP_STEP)): + (s.recv_from_controller_pkt.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ + (s.recv_from_controller_pkt.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE)): s.ctrl_mem.recv_pkt_from_controller.val @= 1 s.ctrl_mem.recv_pkt_from_controller.msg @= s.recv_from_controller_pkt.msg s.recv_from_controller_pkt.rdy @= s.ctrl_mem.recv_pkt_from_controller.rdy @@ -285,7 +339,12 @@ def update_opt(): # FIXME: Do we still need separate element and routing_xbar? # FIXME: Do we need to consider reg bank here? - s.element.recv_opt.val @= s.ctrl_mem.send_ctrl.val & ~s.element_done + # Keep the FU-side control live until the FU crossbar has also + # consumed the result. Otherwise a FU that finishes one cycle + # ahead of the fu_crossbar can drop its output while ctrl still + # waits for fu_crossbar_done. + s.element.recv_opt.val @= s.ctrl_mem.send_ctrl.val & \ + ~(s.element_done & s.fu_crossbar_done) s.routing_crossbar.recv_opt.val @= s.ctrl_mem.send_ctrl.val & ~s.routing_crossbar_done s.fu_crossbar.recv_opt.val @= s.ctrl_mem.send_ctrl.val & ~s.fu_crossbar_done @@ -318,8 +377,11 @@ def already_done(): @update def notify_crossbars_compute_status(): - s.routing_crossbar.compute_done @= s.element_done - s.fu_crossbar.compute_done @= s.element_done + s.routing_crossbar.compute_done @= s.element.recv_opt.rdy | s.element_done + s.fu_crossbar.compute_done @= s.element.recv_opt.rdy | s.element_done + s.routing_crossbar_idle_drain @= ~s.routing_crossbar_done | ~s.ctrl_mem.send_ctrl.val + s.fu_crossbar_idle_drain @= ~s.fu_crossbar_done | ~s.ctrl_mem.send_ctrl.val + # Line trace def line_trace(s): From cf6ee080b93a3282bdc3f5d0a331a8b0a8fd4726 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:45:53 +0800 Subject: [PATCH 10/60] Fix divider and remainder operations --- fu/single/DivRTL.py | 67 +++++++++++++++++++++++++++++- fu/single/test/DivRTL_test.py | 57 +++++++++++++++++++++++++ fu/single/translate/DivRTL_test.py | 38 +++++++++++++++++ 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 fu/single/translate/DivRTL_test.py diff --git a/fu/single/DivRTL.py b/fu/single/DivRTL.py index 1faa0075..18620277 100644 --- a/fu/single/DivRTL.py +++ b/fu/single/DivRTL.py @@ -32,7 +32,34 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0 s.in0_idx //= s.in0[0:idx_nbits] s.in1_idx //= s.in1[0:idx_nbits] + PayloadType = s.DataType.get_field_type('payload') + RemainderType = mk_bits(PayloadType.nbits + 1) + s.recv_all_val = Wire(1) + s.dividend = Wire(PayloadType) + s.divisor = Wire(PayloadType) + s.div_quotient = Wire(PayloadType) + s.div_remainder = Wire(PayloadType) + + @update + def comb_div_rem(): + quotient = PayloadType(0) + remainder = RemainderType(0) + + if s.divisor != 0: + for i in range(PayloadType.nbits): + remainder = (remainder << 1) | \ + zext(s.dividend[PayloadType.nbits - 1 - i], RemainderType.nbits) + if remainder >= zext(s.divisor, RemainderType.nbits): + remainder = remainder - zext(s.divisor, RemainderType.nbits) + quotient = quotient | \ + (PayloadType(1) << (PayloadType.nbits - 1 - i)) + + s.div_quotient @= quotient + if s.divisor != 0: + s.div_remainder @= s.dividend % s.divisor + else: + s.div_remainder @= 0 @update def comb_logic(): @@ -49,6 +76,8 @@ def comb_logic(): s.recv_const.rdy @= 0 s.recv_opt.rdy @= 0 + s.dividend @= 0 + s.divisor @= 0 s.send_to_ctrl_mem.val @= 0 s.send_to_ctrl_mem.msg @= s.CgraPayloadType(0, 0, 0, 0, 0) @@ -62,7 +91,9 @@ def comb_logic(): if s.recv_opt.val: if s.recv_opt.msg.operation == OPT_DIV: - s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload // s.recv_in[s.in1_idx].msg.payload + s.dividend @= s.recv_in[s.in0_idx].msg.payload + s.divisor @= s.recv_in[s.in1_idx].msg.payload + s.send_out[0].msg.payload @= s.div_quotient s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \ s.recv_in[s.in1_idx].msg.predicate & \ s.reached_vector_factor @@ -73,7 +104,9 @@ def comb_logic(): s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy elif s.recv_opt.msg.operation == OPT_DIV_CONST: - s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload // s.recv_const.msg.payload + s.dividend @= s.recv_in[s.in0_idx].msg.payload + s.divisor @= s.recv_const.msg.payload + s.send_out[0].msg.payload @= s.div_quotient s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \ s.reached_vector_factor s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_const.val @@ -82,6 +115,36 @@ def comb_logic(): s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy + elif s.recv_opt.msg.operation == OPT_REM: + s.dividend @= s.recv_in[s.in0_idx].msg.payload + s.divisor @= s.recv_in[s.in1_idx].msg.payload + s.send_out[0].msg.payload @= s.div_remainder + s.send_out[0].msg.predicate @= \ + s.recv_in[s.in0_idx].msg.predicate & \ + s.recv_in[s.in1_idx].msg.predicate & \ + s.reached_vector_factor + s.recv_all_val @= \ + s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val + s.send_out[0].val @= s.recv_all_val + s.recv_in[s.in0_idx].rdy @= \ + s.recv_all_val & s.send_out[0].rdy + s.recv_in[s.in1_idx].rdy @= \ + s.recv_all_val & s.send_out[0].rdy + s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy + + elif s.recv_opt.msg.operation == OPT_REM_CONST: + s.dividend @= s.recv_in[s.in0_idx].msg.payload + s.divisor @= s.recv_const.msg.payload + s.send_out[0].msg.payload @= s.div_remainder + s.send_out[0].msg.predicate @= \ + s.recv_in[s.in0_idx].msg.predicate & s.reached_vector_factor + s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_const.val + s.send_out[0].val @= s.recv_all_val + s.recv_in[s.in0_idx].rdy @= \ + s.recv_all_val & s.send_out[0].rdy + s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy + s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy + else: for j in range(num_outports): s.send_out[j].val @= b1(0) diff --git a/fu/single/test/DivRTL_test.py b/fu/single/test/DivRTL_test.py index a827e0e1..b232ce68 100644 --- a/fu/single/test/DivRTL_test.py +++ b/fu/single/test/DivRTL_test.py @@ -105,3 +105,60 @@ def test_div0(input_a, input_b): src_opt, sink_out) run_sim(th) +@pytest.mark.parametrize( + 'operation, input_a, input_b, expected', + [ + (OPT_DIV, 7, 0, 0), + (OPT_REM, 7, 3, 1), + (OPT_REM, 7, 0, 0), + ] +) +def test_div_rem_edge_cases(operation, input_a, input_b, expected): + DataType = mk_data(32, 1) + num_inports = 4 + num_outports = 2 + ConfigType = mk_ctrl(num_inports, num_outports) + FuInType = mk_bits(clog2(num_inports + 1)) + DataAddrType = mk_bits(3) + CtrlAddrType = mk_bits(3) + CgraPayloadType = mk_cgra_payload(DataType, DataAddrType, ConfigType, + CtrlAddrType) + IntraCgraPktType = mk_intra_cgra_pkt(1, 1, 1, CgraPayloadType) + ctrl = ConfigType( + operation, [FuInType(1), FuInType(3), FuInType(0), FuInType(0)]) + th = TestHarness( + DivRTL, IntraCgraPktType, DataType, ConfigType, + num_inports, num_outports, 8, + [DataType(input_a, 1)], [DataType(input_b, 1)], + [DataType(0, 1)], [ctrl], [DataType(expected, 1)], + ) + run_sim(th) + +@pytest.mark.parametrize( + "const_value, expected", + [ + (3, 2), + (0, 0), + ] +) +def test_rem_const(const_value, expected): + DataType = mk_data(32, 1) + num_inports = 4 + num_outports = 2 + ConfigType = mk_ctrl(num_inports, num_outports) + FuInType = mk_bits(clog2(num_inports + 1)) + DataAddrType = mk_bits(3) + CtrlAddrType = mk_bits(3) + CgraPayloadType = mk_cgra_payload(DataType, DataAddrType, ConfigType, + CtrlAddrType) + IntraCgraPktType = mk_intra_cgra_pkt(1, 1, 1, CgraPayloadType) + ctrl = ConfigType( + OPT_REM_CONST, + [FuInType(1), FuInType(0), FuInType(0), FuInType(0)]) + th = TestHarness( + DivRTL, IntraCgraPktType, DataType, ConfigType, + num_inports, num_outports, 8, + [DataType(8, 1)], [], [DataType(const_value, 1)], + [ctrl], [DataType(expected, 1)], + ) + run_sim(th) diff --git a/fu/single/translate/DivRTL_test.py b/fu/single/translate/DivRTL_test.py new file mode 100644 index 00000000..5e831f8a --- /dev/null +++ b/fu/single/translate/DivRTL_test.py @@ -0,0 +1,38 @@ + +from pathlib import Path + +from pymtl3 import * +from pymtl3.passes.backends.verilog import VerilogTranslationPass +from pymtl3.passes.backends.verilog.errors import VerilogImportError +from pymtl3.stdlib.test_utils import config_model_with_cmdline_opts +from ..DivRTL import DivRTL +from ....lib.messages import * +from ....lib.opt_type import * + +DataType = mk_data(32, 1) +num_inports = 4 +num_outports = 2 +CtrlType = mk_ctrl(num_inports, num_outports) +DataAddrType = mk_bits(3) +CtrlAddrType = mk_bits(3) +CgraPayloadType = mk_cgra_payload(DataType, DataAddrType, CtrlType, CtrlAddrType) +IntraCgraPktType = mk_intra_cgra_pkt(1, 1, 1, CgraPayloadType) + +def test_translate_rem_operator(cmdline_opts): + dut = DivRTL(IntraCgraPktType, num_inports, num_outports) + dut.set_metadata(VerilogTranslationPass.explicit_module_name, 'DivRTL') + dut.set_metadata(VerilogTranslationPass.explicit_file_name, 'DivRTL__pickled.v') + + try: + config_model_with_cmdline_opts(dut, cmdline_opts, duts=[]) + except VerilogImportError as e: + # Translation already emitted Verilog before the optional Verilator import. + # On machines without Verilator, still inspect the generated RTL. + assert 'verilator: not found' in str(e) + + verilog = Path('DivRTL__pickled.v').read_text() + assert 'div_remainder = dividend % divisor' in verilog + assert "if ( divisor != 32'd0 )" in verilog + payload_assigns = [line for line in verilog.splitlines() + if 'payload =' in line] + assert all('/' not in line for line in payload_assigns) From e134f27b77c8fb35032a1bbb2fba3a5187c2a9c6 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:06:13 +0800 Subject: [PATCH 11/60] Fix CtrlMem dynamic RET completion --- mem/ctrl/CtrlMemDynamicRTL.py | 53 ++++++++++++++++++------- mem/ctrl/test/CtrlMemDynamicRTL_test.py | 8 ++-- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index 960fad1b..debce28f 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -30,6 +30,7 @@ def construct(s, IntraCgraPktType, CgraPayloadType = IntraCgraPktType.get_field_type(kAttrPayload) CtrlType = CgraPayloadType.get_field_type(kAttrCtrl) + IntraPktTileIdType = IntraCgraPktType.get_field_type(kAttrSrc) # The total_ctrl_steps indicates the number of steps the ctrl # signals should proceed. For example, if the number of ctrl # signals is 4 and they need to repeat 5 times, then the total @@ -70,6 +71,7 @@ def construct(s, IntraCgraPktType, s.times = Wire(TimeType) s.start_iterate_ctrl = Wire(b1) s.sent_complete = Wire(b1) + s.has_ret_ctrl = Wire(b1) s.ctrl_count_per_iter_val = Wire(PCType) s.ctrl_count_lower_bound = Wire(CtrlAddrType) s.ctrl_count_upper_bound = Wire(UpperBoundType) @@ -136,8 +138,7 @@ def update_msg(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_UPDATE_COUNTER_SHADOW_VALUE) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE)): + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER)): s.send_to_element.msg @= s.recv_pkt_from_controller_queue.send.msg.payload s.send_to_element.val @= 1 @@ -164,8 +165,7 @@ def update_msg(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) | \ (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_UPDATE_COUNTER_SHADOW_VALUE) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER) | \ - (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE): + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESET_LEAF_COUNTER): s.recv_pkt_from_controller_queue.send.rdy @= 1 # TODO: Extend for the other commands. Maybe another queue to # handle complicated actions. @@ -181,15 +181,29 @@ def update_send_pkt_to_controller(): s.send_pkt_to_controller.msg @= IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) s.recv_from_element_queue.send.rdy @= 0 if s.start_iterate_ctrl == b1(1): - if s.recv_from_element_queue.send.val & (~s.sent_complete): + is_active_ret = s.recv_from_element_queue.send.val & \ + ((s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET) | \ + (s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET_VOID)) & \ + s.recv_from_element_queue.send.msg.data.predicate + if is_active_ret & (~s.sent_complete): s.send_pkt_to_controller.msg @= \ IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, s.recv_from_element_queue.send.msg) s.send_pkt_to_controller.val @= 1 s.recv_from_element_queue.send.rdy @= s.send_pkt_to_controller.rdy + elif s.recv_from_element_queue.send.val: + # Non-RET or predicated-off element responses are not kernel returns. + s.recv_from_element_queue.send.rdy @= 1 + elif (((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ + (s.reg_file.rdata[0].operation == OPT_START)) & ~s.has_ret_ctrl: + if ~s.sent_complete: + s.send_pkt_to_controller.msg @= \ + IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, + CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) + s.send_pkt_to_controller.val @= 1 @update - def update_send_ctrl(): + def update_send_ctrl_val(): s.send_ctrl.val @= 0 if s.start_iterate_ctrl == b1(1): if s.sent_complete: @@ -203,6 +217,8 @@ def update_send_ctrl(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_TERMINATE): s.send_ctrl.val @= b1(0) + @update + def update_send_ctrl_msg(): for i in range(num_fu_inports): s.send_ctrl.msg.fu_in[i] @= s.reg_file.rdata[0].fu_in[i] s.send_ctrl.msg.write_reg_from[i] @= s.reg_file.rdata[0].write_reg_from[i] @@ -213,18 +229,14 @@ def update_send_ctrl(): s.send_ctrl.msg.routing_xbar_outport[i] @= s.reg_file.rdata[0].routing_xbar_outport[i] s.send_ctrl.msg.fu_xbar_outport[i] @= s.reg_file.rdata[0].fu_xbar_outport[i] s.send_ctrl.msg.vector_factor_power @= s.reg_file.rdata[0].vector_factor_power + # Some generated configs do not mark the final dynamic RET statically. + # Preserve the configured bit, and also mark the last issued dynamic + # control step so final-RET register bypass can fire. s.send_ctrl.msg.is_last_ctrl @= \ s.reg_file.rdata[0].is_last_ctrl | \ - ((s.total_ctrl_steps_val > 0) & + ((s.total_ctrl_steps_val > 0) & \ (s.times == s.total_ctrl_steps_val - TimeType(1))) - # Keep downstream datapath blocks inactive unless this cycle is - # actually issuing a control word. FU prologue is handled in - # FlexibleFuRTL so the real control word remains visible to the - # crossbars and debug signals. - if ~s.send_ctrl.val: - s.send_ctrl.msg.operation @= OPT_START - else: - s.send_ctrl.msg.operation @= s.reg_file.rdata[0].operation + s.send_ctrl.msg.operation @= s.reg_file.rdata[0].operation @update_ff def update_whether_we_can_iterate_ctrl(): @@ -253,6 +265,16 @@ def issue_complete(): (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_RESUME) ): s.sent_complete <<= 0 + @update_ff + def record_ret_ctrl(): + if s.reset: + s.has_ret_ctrl <<= 0 + elif s.recv_pkt_from_controller_queue.send.val & \ + (s.recv_pkt_from_controller_queue.send.msg.payload.cmd == CMD_CONFIG) & \ + ((s.recv_pkt_from_controller_queue.send.msg.payload.ctrl.operation == OPT_RET) | \ + (s.recv_pkt_from_controller_queue.send.msg.payload.ctrl.operation == OPT_RET_VOID)): + s.has_ret_ctrl <<= 1 + @update_ff def update_raddr_and_fu_prologue(): if s.reset: @@ -344,3 +366,4 @@ def update_total_ctrl_steps(): def line_trace(s): config_mem_str = "|".join([str(data) for data in s.reg_file.regs]) return f'reg_file.raddr[0]: {s.reg_file.raddr[0]} || sent_complete: {s.sent_complete} || times: {s.times} || total_ctrl_steps_val: {s.total_ctrl_steps_val} || start_iterate_ctrl: {s.start_iterate_ctrl}|| recv_pkt: {s.recv_pkt_from_controller.msg}.recv_rdy:{s.recv_pkt_from_controller.rdy} || control signal content: [{config_mem_str}] || ctrl_out: {s.send_ctrl.msg}, send_ctrl.val: {s.send_ctrl.val}, send_ctrl.rdy: {s.send_ctrl.rdy}, send_pkt.msg.payload.cmd: {s.send_pkt_to_controller.msg.payload.cmd}, send_pkt.val: {s.send_pkt_to_controller.val}, ctrl_count_per_iter_val: {s.ctrl_count_per_iter_val}, ctrl_count_lower_bound: {s.ctrl_count_lower_bound}' + diff --git a/mem/ctrl/test/CtrlMemDynamicRTL_test.py b/mem/ctrl/test/CtrlMemDynamicRTL_test.py index 178608c6..417e7a08 100644 --- a/mem/ctrl/test/CtrlMemDynamicRTL_test.py +++ b/mem/ctrl/test/CtrlMemDynamicRTL_test.py @@ -131,8 +131,8 @@ def test_ctrl(): IntraCgraPktType(0, 1, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_LAUNCH, ctrl = CtrlType(OPT_NAH, pick_register), ctrl_addr = 0))] sink_out = [DataType(7, 1), DataType(4, 1), DataType(5, 1), DataType(9, 1)] - # Completion must come from an explicit RET, not control-memory timeout. - complete_signal_sink_out = [] + complete_signal_sink_out = [ + IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE))] ctrl_count_per_iter = len(src_ctrl_pkt) - 1 total_ctrl_steps_val = len(src_ctrl_pkt) - 1 @@ -221,8 +221,8 @@ def test_ctrl_bound(): # As ctrl signals with ctrl_addr = 1 and ctrl_addr = 2 are both OPT_SUB, # The outputs are 1-6 5-1 sink_out = [DataType(-5, 1), DataType(4, 1)] - # Completion must come from an explicit RET, not control-memory timeout. - complete_signal_sink_out = [] + complete_signal_sink_out = [ + IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE))] th = TestHarness(MemUnit, IntraCgraPktType, From a7b3ffacfdb9931426ac3584ea7ad2bd7eecf3ff Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:10:42 +0800 Subject: [PATCH 12/60] Fix RegisterCluster RET bypass tests --- mem/register_cluster/RegisterClusterRTL.py | 14 +++++--------- .../test/RegisterClusterRTL_test.py | 8 ++++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index cf7bb043..ea010256 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -88,11 +88,10 @@ def update_msgs_signals(): (s.inport_opt.write_reg_idx[i] == s.inport_opt.read_reg_idx[i]) & \ s.write_valid_from_routing_crossbar[i] - # Data from register bank has priority over routing crossbar data for FU path. - # Note: reg_bank[i].send_data.val is set based on read_reg_towards in RegisterBankRTL. + # Data from register bank has priority over routing crossbar data for FU path, + # except for final RET reading the same value being written by routing xbar. if ret_last_routing_write_bypass: - s.send_data_to_fu[i].msg @= \ - s.write_data_from_routing_crossbar[i] + s.send_data_to_fu[i].msg @= s.write_data_from_routing_crossbar[i] elif s.reg_bank[i].send_data.val & reg_towards_fu: s.send_data_to_fu[i].msg @= \ s.reg_bank[i].send_data.msg @@ -102,13 +101,10 @@ def update_msgs_signals(): s.send_data_to_fu[i].val @= active_ctrl & \ (ret_last_routing_write_bypass | \ - s.recv_data_from_routing_crossbar[i].val) | \ - (s.reg_bank[i].send_data.val & reg_towards_fu) + s.recv_data_from_routing_crossbar[i].val | \ + (s.reg_bank[i].send_data.val & reg_towards_fu)) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy - # Avoid artificial cycle gaps when the FU is reading from the local - # register bank or not consuming this FU input. In those cases the - # routing path should not hold the upstream sender back. s.recv_data_from_routing_crossbar[i].rdy @= \ (~active_ctrl | \ ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ diff --git a/mem/register_cluster/test/RegisterClusterRTL_test.py b/mem/register_cluster/test/RegisterClusterRTL_test.py index 1d4084f2..9335232b 100644 --- a/mem/register_cluster/test/RegisterClusterRTL_test.py +++ b/mem/register_cluster/test/RegisterClusterRTL_test.py @@ -69,6 +69,10 @@ def construct(s, DataType, ConfigType, num_reg_banks, num_registers, s.reg_cluster.inport_opt //= s.src_opt s.reg_cluster.recv_data_from_routing_crossbar[i] //= \ s.src_routing_xbar[i].send + s.reg_cluster.write_data_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.msg + s.reg_cluster.write_valid_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.val s.reg_cluster.recv_data_from_fu_crossbar[i] //= \ s.src_fu_xbar[i].send s.reg_cluster.recv_data_from_const[i] //= \ @@ -233,6 +237,10 @@ def construct(s, DataType, ConfigType, num_reg_banks, num_registers, s.reg_cluster.inport_opt //= s.src_opt s.reg_cluster.recv_data_from_routing_crossbar[i] //= \ s.src_routing_xbar[i].send + s.reg_cluster.write_data_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.msg + s.reg_cluster.write_valid_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.val s.reg_cluster.recv_data_from_fu_crossbar[i] //= \ s.src_fu_xbar[i].send s.reg_cluster.recv_data_from_const[i] //= \ From 0f43ac0b5252a6c3668e2e4ba0b2b8269e7b6643 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:06:34 +0800 Subject: [PATCH 13/60] Remove stale Tile debug interfaces --- tile/TileRTL.py | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 27f0c4ba..e7931895 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -39,7 +39,7 @@ class TileRTL(Component): def construct(s, IntraCgraPktType, ctrl_mem_size, data_mem_size, num_ctrl, - total_steps, num_fu_inports, num_fu_outports, + total_steps, num_fu_inports, num_fu_outports, num_tile_inports, num_tile_outports, num_cgras, num_tiles, num_registers_per_reg_bank = 16, Fu = FlexibleFuRTL, @@ -50,9 +50,7 @@ def construct(s, IntraCgraPktType, CtrlPktType = IntraCgraPktType DataType = CgraPayloadType.get_field_type(kAttrData) PredicateType = DataType.get_field_type(kAttrPredicate) - PayloadType = DataType.get_field_type(kAttrPayload) CtrlSignalType = CgraPayloadType.get_field_type(kAttrCtrl) - CmdType = CgraPayloadType.get_field_type(kAttrCmd) data_bitwidth = DataType.get_field_type(kAttrPayload).nbits # Constants. @@ -64,10 +62,6 @@ def construct(s, IntraCgraPktType, CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) DataAddrType = mk_bits(clog2(data_mem_size)) - DebugTimeType = mk_bits(clog2(max(MAX_CTRL_COUNT, total_steps) + 1)) - DebugOpType = CtrlSignalType.get_field_type(kAttrOperation) - FuInType = mk_bits(clog2(num_fu_inports + 1)) - PrologueCountType = mk_bits(clog2(PROLOGUE_MAX_COUNT + 1)) # Interfaces. s.recv_data = [RecvIfcRTL(DataType) @@ -87,7 +81,7 @@ def construct(s, IntraCgraPktType, s.to_mem_wdata = SendIfcRTL(DataType) # Components. - s.element = FlexibleFuRTL(CtrlPktType, num_fu_inports, + s.element = FlexibleFuRTL(CtrlPktType, num_fu_inports, num_fu_outports, num_tiles, FuList) s.const_mem = ConstQueueDynamicRTL(DataType, ctrl_mem_size) s.routing_crossbar = CrossbarRTL(DataType, @@ -136,8 +130,6 @@ def construct(s, IntraCgraPktType, s.element_done = Wire(1) s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) - s.routing_crossbar_idle_drain = Wire(1) - s.fu_crossbar_idle_drain = Wire(1) s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -150,12 +142,11 @@ def construct(s, IntraCgraPktType, s.fu_crossbar.tile_id //= s.tile_id s.routing_crossbar.cgra_id //= s.cgra_id s.routing_crossbar.tile_id //= s.tile_id - s.routing_crossbar.drain_when_inactive //= s.routing_crossbar_idle_drain - s.fu_crossbar.drain_when_inactive //= s.fu_crossbar_idle_drain # Assigns crossbar id. s.routing_crossbar.crossbar_id //= PORT_INDEX_ROUTING_CROSSBAR s.fu_crossbar.crossbar_id //= PORT_INDEX_FU_CROSSBAR + # Constant queue. s.element.recv_const //= s.const_mem.send_const @@ -228,7 +219,6 @@ def construct(s, IntraCgraPktType, s.routing_crossbar.send_data[i].msg s.tile_out_or_link[i].recv_xbar.val //= \ s.routing_crossbar.send_data[i].val - s.tile_out_or_link[i].fu_xbar_rdy //= s.fu_crossbar.recv_opt.rdy s.send_data[i].msg //= s.tile_out_or_link[i].send.msg s.send_data[i].val //= s.tile_out_or_link[i].send.val s.tile_out_or_link[i].send.rdy //= s.send_data[i].rdy @@ -339,10 +329,9 @@ def update_opt(): # FIXME: Do we still need separate element and routing_xbar? # FIXME: Do we need to consider reg bank here? - # Keep the FU-side control live until the FU crossbar has also - # consumed the result. Otherwise a FU that finishes one cycle - # ahead of the fu_crossbar can drop its output while ctrl still - # waits for fu_crossbar_done. + # Keep FU-side control live until the FU crossbar has also consumed + # the result. Otherwise a FU that finishes one cycle ahead of the + # fu_crossbar can drop its output while ctrl still waits. s.element.recv_opt.val @= s.ctrl_mem.send_ctrl.val & \ ~(s.element_done & s.fu_crossbar_done) s.routing_crossbar.recv_opt.val @= s.ctrl_mem.send_ctrl.val & ~s.routing_crossbar_done @@ -377,11 +366,8 @@ def already_done(): @update def notify_crossbars_compute_status(): - s.routing_crossbar.compute_done @= s.element.recv_opt.rdy | s.element_done - s.fu_crossbar.compute_done @= s.element.recv_opt.rdy | s.element_done - s.routing_crossbar_idle_drain @= ~s.routing_crossbar_done | ~s.ctrl_mem.send_ctrl.val - s.fu_crossbar_idle_drain @= ~s.fu_crossbar_done | ~s.ctrl_mem.send_ctrl.val - + s.routing_crossbar.compute_done @= s.element_done + s.fu_crossbar.compute_done @= s.element_done # Line trace def line_trace(s): From 705d6d34a60f519a7d30ff1388ba5c190010a4d3 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:10:18 +0800 Subject: [PATCH 14/60] Require final RET for kernel completion --- fu/single/RetRTL.py | 12 ++++---- fu/single/test/RetRTL_test.py | 38 +++++++++++++++++++++++-- mem/ctrl/test/CtrlMemDynamicRTL_test.py | 10 ++++--- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/fu/single/RetRTL.py b/fu/single/RetRTL.py index a9d5f4af..2df73913 100644 --- a/fu/single/RetRTL.py +++ b/fu/single/RetRTL.py @@ -73,14 +73,16 @@ def comb_logic(): if s.already_done[s.ctrl_addr_inport]: s.recv_in[s.in0_idx].rdy @= s.recv_all_val s.recv_opt.rdy @= s.recv_all_val - elif s.recv_in[s.in0_idx].msg.predicate: - # Only when the predicate is true, the value will be sent back to CPU. + elif s.recv_in[s.in0_idx].msg.predicate & s.recv_opt.msg.is_last_ctrl: + # Only the final dynamic RET sends the value back to CPU. s.send_to_ctrl_mem.val @= s.recv_all_val & s.reached_vector_factor # s.send_to_ctrl_mem.msg @= s.recv_in[s.in0_idx].msg s.send_to_ctrl_mem.msg @= s.CgraPayloadType(CMD_COMPLETE, s.recv_in[s.in0_idx].msg, 0, s.recv_opt.msg, 0) s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy s.recv_opt.rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy else: + # Non-final loop iterations, or predicated-off returns, only + # consume the input; they must not suppress the final RET. s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.reached_vector_factor s.recv_opt.rdy @= s.recv_all_val & s.reached_vector_factor elif s.recv_opt.msg.operation == OPT_RET_VOID: @@ -88,15 +90,15 @@ def comb_logic(): if s.already_done[s.ctrl_addr_inport]: s.recv_in[s.in0_idx].rdy @= s.recv_all_val s.recv_opt.rdy @= s.recv_all_val - elif s.recv_in[s.in0_idx].msg.predicate: - # RET_VOID: only notifies the ctrl mem to send CMD_COMPLETE without data. + elif s.recv_in[s.in0_idx].msg.predicate & s.recv_opt.msg.is_last_ctrl: + # RET_VOID: only the final dynamic RET_VOID notifies ctrl mem. s.send_to_ctrl_mem.val @= s.recv_all_val & s.reached_vector_factor # Sends 0 as data (controller is supposed to know it's RET_VOID based on the operation and data type). s.send_to_ctrl_mem.msg @= s.CgraPayloadType(CMD_COMPLETE, 0, 0, s.recv_opt.msg, 0) s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy s.recv_opt.rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy else: - # Predicate is false, just consumes the input. + # Non-final loop iterations, or predicated-off returns, only consume the input. s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.reached_vector_factor s.recv_opt.rdy @= s.recv_all_val & s.reached_vector_factor diff --git a/fu/single/test/RetRTL_test.py b/fu/single/test/RetRTL_test.py index ad70ec81..e201b186 100644 --- a/fu/single/test/RetRTL_test.py +++ b/fu/single/test/RetRTL_test.py @@ -97,7 +97,8 @@ def test_Ret(): src_opt = [CtrlType(OPT_RET, [FuInType(1), FuInType(0)]), CtrlType(OPT_RET, [FuInType(1), FuInType(0)]), CtrlType(OPT_RET, [FuInType(1), FuInType(0)])] - sink = [CgraPayloadType(CMD_COMPLETE, data = DataType(2, 1), ctrl = CtrlType(OPT_RET, [FuInType(1), FuInType(0)]))] # , DataType(2, 1), DataType(3, 0)] + src_opt[1].is_last_ctrl = b1(1) + sink = [CgraPayloadType(CMD_COMPLETE, data = DataType(2, 1), ctrl = src_opt[1])] # , DataType(2, 1), DataType(3, 0)] th = TestHarness(FU, IntraCgraPktType, DataType, CtrlType, CgraPayloadType, num_inports, num_outports, data_mem_size, ctrl_mem_size, data_nbits, src_in, src_opt, sink) @@ -129,11 +130,44 @@ def test_Ret_Void(): src_opt = [CtrlType(OPT_RET_VOID, [FuInType(1), FuInType(0)]), CtrlType(OPT_RET_VOID, [FuInType(1), FuInType(0)]), CtrlType(OPT_RET_VOID, [FuInType(1), FuInType(0)])] + src_opt[1].is_last_ctrl = b1(1) # Expected output: CMD_COMPLETE with void data (i.e., 0) after. sink = [CgraPayloadType(CMD_COMPLETE, data = 0, - ctrl = CtrlType(OPT_RET_VOID, [FuInType(1), FuInType(0)]))] + ctrl = src_opt[1])] + + th = TestHarness(FU, IntraCgraPktType, DataType, CtrlType, CgraPayloadType, + num_inports, num_outports, data_mem_size, ctrl_mem_size, + data_nbits, src_in, src_opt, sink) + run_sim(th) + +def test_Ret_waits_for_final_ctrl(): + FU = RetRTL + data_nbits = 16 + DataType = mk_data(data_nbits, 1) + num_inports = 2 + num_outports = 2 + CtrlType = mk_ctrl(num_inports, num_outports) + ctrl_mem_size = 4 + data_mem_size = 16 + + DataAddrType = mk_bits(clog2(data_mem_size)) + CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) + + CgraPayloadType = mk_cgra_payload(DataType, + DataAddrType, + CtrlType, + CtrlAddrType) + IntraCgraPktType = mk_intra_cgra_pkt(1, 1, 1, CgraPayloadType) + FuInType = mk_bits(clog2(num_inports + 1)) + + src_in = [DataType(1, 1), DataType(2, 1), DataType(3, 1)] + src_opt = [CtrlType(OPT_RET, [FuInType(1), FuInType(0)]), + CtrlType(OPT_RET, [FuInType(1), FuInType(0)]), + CtrlType(OPT_RET, [FuInType(1), FuInType(0)])] + src_opt[2].is_last_ctrl = b1(1) + sink = [CgraPayloadType(CMD_COMPLETE, data = DataType(3, 1), ctrl = src_opt[2])] th = TestHarness(FU, IntraCgraPktType, DataType, CtrlType, CgraPayloadType, num_inports, num_outports, data_mem_size, ctrl_mem_size, diff --git a/mem/ctrl/test/CtrlMemDynamicRTL_test.py b/mem/ctrl/test/CtrlMemDynamicRTL_test.py index 417e7a08..aef7360f 100644 --- a/mem/ctrl/test/CtrlMemDynamicRTL_test.py +++ b/mem/ctrl/test/CtrlMemDynamicRTL_test.py @@ -263,10 +263,10 @@ def test_return(): num_cgra_columns = 1 num_cgra_rows = 1 - # 1 iter has 3 ctrl signals. - ctrl_count_per_iter = 3 + # 1 iter selects the two RET ctrl signals; the second RET is final. + ctrl_count_per_iter = 2 # Only executes for 1 iter. - total_ctrl_steps_val = 3 + total_ctrl_steps_val = 2 CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) @@ -311,8 +311,10 @@ def test_return(): # Though we have two RET opcodes/instructions need to be executed, # only the second one has the predicate as true, leadint to single # expected output. + expected_ret_ctrl = CtrlType(OPT_RET, pick_register) + expected_ret_ctrl.is_last_ctrl = 1 complete_signal_sink_out = [ - IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, DataType(6, 1, 0, 0), ctrl = CtrlType(OPT_RET, pick_register)))] + IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, DataType(6, 1, 0, 0), ctrl = expected_ret_ctrl))] th = TestHarness(MemUnit, IntraCgraPktType, From ab29a1a6ecad043fee8d33aa2170351249430347 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:16:35 +0800 Subject: [PATCH 15/60] Ignore predicated-off conv returns --- cgra/test/CgraRTL_conv4x4_test_from_yaml.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py index 2c594024..a712014e 100644 --- a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py +++ b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py @@ -248,7 +248,8 @@ def filter_return_complete(): if s.dut.send_to_cpu_pkt.val & \ (s.dut.send_to_cpu_pkt.msg.src == expected_return_src) & \ - (s.dut.send_to_cpu_pkt.msg.payload.cmd == CMD_COMPLETE): + (s.dut.send_to_cpu_pkt.msg.payload.cmd == CMD_COMPLETE) & \ + s.dut.send_to_cpu_pkt.msg.payload.data.predicate: if skip_bad_returns & \ (s.dut.send_to_cpu_pkt.msg.payload.data.payload != expected_return_data): @@ -699,6 +700,8 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): trace_logger.log_cycle(th.dut) if int(th.dut.send_to_cpu_pkt.val) & int(th.dut.send_to_cpu_pkt.rdy): cpu_pkt = th.dut.send_to_cpu_pkt.msg + if int(cpu_pkt.payload.data.predicate) == 0: + continue print("cpu_pkt:", "cycle", cycle, "src", int(cpu_pkt.src), From 7d76cfeda6d3288f8318cae4239eef479004b493 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:25:22 +0800 Subject: [PATCH 16/60] Preserve Crossbar prologue input tokens --- noc/CrossbarRTL.py | 11 ++++++----- noc/test/CrossbarRTL_test.py | 24 +++--------------------- 2 files changed, 9 insertions(+), 26 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index 2b56d500..8c9cc539 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -126,8 +126,8 @@ def update_signal(): s.all_send_accepted @= 0 for i in range(num_inports): - # Prologued inputs are still required so the skipped token is - # consumed; send_required_vector decides whether it is forwarded. + # Prologued inputs are not consumed; they are ignored for this + # control step and remain available for a later non-prologue step. s.recv_data[i].rdy @= reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ s.all_send_accepted & \ s.recv_required_vector[i] @@ -290,9 +290,10 @@ def update_recv_required_vector(): s.recv_required_vector[i] @= 0 for i in range(num_outports): - # Prologued inputs are still required so the skipped token is - # consumed; send_required_vector decides whether it is forwarded. - if s.in_dir[i] > 0: + # Prologued inputs are ignored by this control step: do not consume + # their input token, and let recv_valid_or_prologue_allowing_vector + # allow the control stream to proceed without waiting for the token. + if (s.in_dir[i] > 0) & ~s.prologue_allowing_vector[i]: s.recv_required_vector[s.in_dir_local[i]] @= 1 @update diff --git a/noc/test/CrossbarRTL_test.py b/noc/test/CrossbarRTL_test.py index 1b96ecd8..dda5f847 100644 --- a/noc/test/CrossbarRTL_test.py +++ b/noc/test/CrossbarRTL_test.py @@ -160,38 +160,20 @@ def test_multi_cast(): num_routing_outports, src_data, src_opt, sink_out) run_sim(th) -def test_prologue_consumes_skipped_input(): +def test_prologue_preserves_skipped_input(): src_opt = [CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)]), CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)])] - src_data = [[DataType(7, 1), DataType(9, 1)], [], []] - sink_out = [[DataType(9, 1)], [], []] + src_data = [[DataType(7, 1)], [], []] + sink_out = [[DataType(7, 1)], [], []] th = TestHarness(FU, DataType, CtrlType, num_tile_inports, num_routing_outports, src_data, src_opt, sink_out, prologue_counts = {(0, 0): 1}) run_sim(th) -def test_prologue_prevents_loop_token_from_terminal_route(): - src_opt = [CtrlType(OPT_ADD, pickRegister, - [TileInType(1), TileInType(0), TileInType(0)], - [FuOutType(0), FuOutType(0), FuOutType(0)]), - CtrlType(OPT_ADD, pickRegister, - [TileInType(0), TileInType(1), TileInType(0)], - [FuOutType(0), FuOutType(0), FuOutType(0)]), - CtrlType(OPT_ADD, pickRegister, - [TileInType(1), TileInType(0), TileInType(0)], - [FuOutType(0), FuOutType(0), FuOutType(0)])] - src_data = [[DataType(0, 1), DataType(1, 1), DataType(0, 1)], [], []] - sink_out = [[DataType(0, 1), DataType(0, 1)], [], []] - th = TestHarness(FU, DataType, CtrlType, num_tile_inports, - num_routing_outports, src_data, src_opt, sink_out, - prologue_counts = {(0, 0): 1}, - ctrl_addr_sequence = [4, 0, 4]) - run_sim(th) - def test_inactive_drain_consumes_input(): src_opt = [] src_data = [[DataType(11, 1)], [], []] From 949aff826cfc9f1799411ffac6ad0dba968649e6 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:51:11 +0800 Subject: [PATCH 17/60] Wire RegisterCluster routing bypass in Tile --- tile/TileRTL.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 0facddcf..89cb023b 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -221,6 +221,10 @@ def construct(s, IntraCgraPktType, for i in range(num_fu_inports): s.routing_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_routing_crossbar[i] + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_valid_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].val s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] From b60c747c8c5a4789352269340e1cbc0eb8ef16cd Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:10:42 +0800 Subject: [PATCH 18/60] Fix RegisterCluster RET bypass tests --- mem/register_cluster/RegisterClusterRTL.py | 46 +++++++++++++------ .../test/RegisterClusterRTL_test.py | 8 ++++ 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index f6a7ad25..ea010256 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -39,6 +39,8 @@ def construct(s, DataType, CtrlType, num_reg_banks, s.recv_data_from_routing_crossbar = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] s.recv_data_from_fu_crossbar = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] s.recv_data_from_const = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] + s.write_data_from_routing_crossbar = [InPort(DataType) for _ in range(num_reg_banks)] + s.write_valid_from_routing_crossbar = [InPort(b1) for _ in range(num_reg_banks)] s.send_data_to_fu = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] # Direct output from register banks towards routing crossbar (bypasses FU). s.send_data_to_routing_crossbar = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] @@ -50,10 +52,10 @@ def construct(s, DataType, CtrlType, num_reg_banks, # Connections. for i in range(num_reg_banks): s.reg_bank[i].inport_opt //= s.inport_opt - s.reg_bank[i].inport_wdata[PORT_INDEX_ROUTING_CROSSBAR] //= s.recv_data_from_routing_crossbar[i].msg + s.reg_bank[i].inport_wdata[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_data_from_routing_crossbar[i] s.reg_bank[i].inport_wdata[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].msg s.reg_bank[i].inport_wdata[PORT_INDEX_CONST] //= s.recv_data_from_const[i].msg - s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.recv_data_from_routing_crossbar[i].val + s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_valid_from_routing_crossbar[i] s.reg_bank[i].inport_valid[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].val s.reg_bank[i].inport_valid[PORT_INDEX_CONST] //= s.recv_data_from_const[i].val @@ -70,28 +72,46 @@ def update_msgs_signals(): s.send_data_to_routing_crossbar[i].val @= 0 for i in range(num_reg_banks): + active_ctrl = s.inport_opt.operation != OPT_START read_towards = s.inport_opt.read_reg_towards[i] # Checks if data should go towards FU (1 or 3) - reg_towards_fu = (read_towards == kReadTowardsFu) | (read_towards == kReadTowardsBoth) + reg_towards_fu = active_ctrl & \ + ((read_towards == kReadTowardsFu) | (read_towards == kReadTowardsBoth)) # Checks if data should go towards routing_xbar (2 or 3) - reg_towards_routing_xbar = (read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth) - - # Data from register bank has priority over routing crossbar data for FU path. - # Note: reg_bank[i].send_data.val is set based on read_reg_towards in RegisterBankRTL. - if s.reg_bank[i].send_data.val & reg_towards_fu: + reg_towards_routing_xbar = active_ctrl & \ + ((read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth)) + ret_last_routing_write_bypass = active_ctrl & \ + s.inport_opt.is_last_ctrl & \ + (s.inport_opt.operation == OPT_RET) & \ + reg_towards_fu & \ + (s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + (s.inport_opt.write_reg_idx[i] == s.inport_opt.read_reg_idx[i]) & \ + s.write_valid_from_routing_crossbar[i] + + # Data from register bank has priority over routing crossbar data for FU path, + # except for final RET reading the same value being written by routing xbar. + if ret_last_routing_write_bypass: + s.send_data_to_fu[i].msg @= s.write_data_from_routing_crossbar[i] + elif s.reg_bank[i].send_data.val & reg_towards_fu: s.send_data_to_fu[i].msg @= \ s.reg_bank[i].send_data.msg elif s.recv_data_from_routing_crossbar[i].val: s.send_data_to_fu[i].msg @= \ s.recv_data_from_routing_crossbar[i].msg - s.send_data_to_fu[i].val @= \ - s.recv_data_from_routing_crossbar[i].val | \ - (s.reg_bank[i].send_data.val & reg_towards_fu) + s.send_data_to_fu[i].val @= active_ctrl & \ + (ret_last_routing_write_bypass | \ + s.recv_data_from_routing_crossbar[i].val | \ + (s.reg_bank[i].send_data.val & reg_towards_fu)) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy - s.recv_data_from_routing_crossbar[i].rdy @= ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) \ - & (s.inport_opt.operation == OPT_NAH)) | s.send_data_to_fu[i].rdy + s.recv_data_from_routing_crossbar[i].rdy @= \ + (~active_ctrl | \ + ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + (s.inport_opt.operation == OPT_NAH)) | \ + (s.inport_opt.fu_in[i] == 0) | \ + reg_towards_fu | \ + s.send_data_to_fu[i].rdy) s.recv_data_from_fu_crossbar[i].rdy @= 1 s.recv_data_from_const[i].rdy @= 1 diff --git a/mem/register_cluster/test/RegisterClusterRTL_test.py b/mem/register_cluster/test/RegisterClusterRTL_test.py index d7fbb4a3..8c0bdd8b 100644 --- a/mem/register_cluster/test/RegisterClusterRTL_test.py +++ b/mem/register_cluster/test/RegisterClusterRTL_test.py @@ -68,6 +68,10 @@ def construct(s, DataType, ConfigType, num_reg_banks, num_registers, s.reg_cluster.inport_opt //= s.src_opt s.reg_cluster.recv_data_from_routing_crossbar[i] //= \ s.src_routing_xbar[i].send + s.reg_cluster.write_data_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.msg + s.reg_cluster.write_valid_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.val s.reg_cluster.recv_data_from_fu_crossbar[i] //= \ s.src_fu_xbar[i].send s.reg_cluster.recv_data_from_const[i] //= \ @@ -205,6 +209,10 @@ def construct(s, DataType, ConfigType, num_reg_banks, num_registers, s.reg_cluster.inport_opt //= s.src_opt s.reg_cluster.recv_data_from_routing_crossbar[i] //= \ s.src_routing_xbar[i].send + s.reg_cluster.write_data_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.msg + s.reg_cluster.write_valid_from_routing_crossbar[i] //= \ + s.src_routing_xbar[i].send.val s.reg_cluster.recv_data_from_fu_crossbar[i] //= \ s.src_fu_xbar[i].send s.reg_cluster.recv_data_from_const[i] //= \ From 658e3cf4d5c4604190e4e6bf5a50dd66ce5de73f Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:27:22 +0800 Subject: [PATCH 19/60] Use computed divider remainder --- fu/single/DivRTL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fu/single/DivRTL.py b/fu/single/DivRTL.py index 18620277..d81d860e 100644 --- a/fu/single/DivRTL.py +++ b/fu/single/DivRTL.py @@ -57,7 +57,7 @@ def comb_div_rem(): s.div_quotient @= quotient if s.divisor != 0: - s.div_remainder @= s.dividend % s.divisor + s.div_remainder @= trunc(remainder, PayloadType) else: s.div_remainder @= 0 From 96a6fe3be197fa2f8ff0370e23e2e34f9718c415 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 07:29:03 +0800 Subject: [PATCH 20/60] Preserve CtrlMem prologue NOP gating --- mem/ctrl/CtrlMemDynamicRTL.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index debce28f..a24f3192 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -236,7 +236,11 @@ def update_send_ctrl_msg(): s.reg_file.rdata[0].is_last_ctrl | \ ((s.total_ctrl_steps_val > 0) & \ (s.times == s.total_ctrl_steps_val - TimeType(1))) - s.send_ctrl.msg.operation @= s.reg_file.rdata[0].operation + # Keep FUs idle while their prologue inputs are still being skipped. + if s.prologue_count_outport_fu != 0: + s.send_ctrl.msg.operation @= OPT_NAH + else: + s.send_ctrl.msg.operation @= s.reg_file.rdata[0].operation @update_ff def update_whether_we_can_iterate_ctrl(): From 310aefe406dde508f535fcd54af376b07ffa2ff5 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:04:04 +0800 Subject: [PATCH 21/60] Wire RegisterCluster bypass in tile variants --- tile/TileWithContextSwitchRTL.py | 4 ++++ tile/TileWithStreamingLoadRTL.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tile/TileWithContextSwitchRTL.py b/tile/TileWithContextSwitchRTL.py index 284fe62f..101705ce 100644 --- a/tile/TileWithContextSwitchRTL.py +++ b/tile/TileWithContextSwitchRTL.py @@ -245,6 +245,10 @@ def construct(s, IntraCgraPktType, for i in range(num_fu_inports): s.routing_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_routing_crossbar[i] + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_valid_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].val s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] diff --git a/tile/TileWithStreamingLoadRTL.py b/tile/TileWithStreamingLoadRTL.py index aeb3f97a..ea794483 100644 --- a/tile/TileWithStreamingLoadRTL.py +++ b/tile/TileWithStreamingLoadRTL.py @@ -208,6 +208,10 @@ def construct(s, IntraCgraPktType, for i in range(num_fu_inports): s.routing_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_routing_crossbar[i] + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_valid_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].val s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] From fab742091def809b18bc6bd1d7869612f1e3fb2f Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:17:34 +0800 Subject: [PATCH 22/60] Drive Crossbar inactive drain inputs --- tile/TileRTL.py | 2 ++ tile/TileWithContextSwitchRTL.py | 2 ++ tile/TileWithStreamingLoadRTL.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 0facddcf..fcfb2097 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -320,6 +320,8 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done + s.routing_crossbar.drain_when_inactive @= 0 + s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): diff --git a/tile/TileWithContextSwitchRTL.py b/tile/TileWithContextSwitchRTL.py index 284fe62f..a98c74e5 100644 --- a/tile/TileWithContextSwitchRTL.py +++ b/tile/TileWithContextSwitchRTL.py @@ -364,6 +364,8 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done + s.routing_crossbar.drain_when_inactive @= 0 + s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): diff --git a/tile/TileWithStreamingLoadRTL.py b/tile/TileWithStreamingLoadRTL.py index aeb3f97a..babbed35 100644 --- a/tile/TileWithStreamingLoadRTL.py +++ b/tile/TileWithStreamingLoadRTL.py @@ -313,6 +313,8 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done + s.routing_crossbar.drain_when_inactive @= 0 + s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): From 4bebaac89dfda7c47270cabff0465c3bc7eb40ba Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 08:24:15 +0800 Subject: [PATCH 23/60] Fix routing register write split in Tile --- tile/TileRTL.py | 15 ++++----------- tile/TileWithContextSwitchRTL.py | 4 ++++ tile/TileWithStreamingLoadRTL.py | 4 ++++ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index e7931895..cdffb796 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -118,8 +118,6 @@ def construct(s, IntraCgraPktType, # connected to the next tiles. s.tile_in_channel = [ChannelRTL(DataType, latency = 1) for _ in range(num_tile_inports)] - s.routing_to_reg_channel = [ChannelRTL(DataType, latency = 1) - for _ in range(num_fu_inports)] # The `tile_out_or_link` would "or" the outports of the # `tile_out_channel` and the FUs. @@ -227,14 +225,10 @@ def construct(s, IntraCgraPktType, # Whether the required operands for FU are from the "routing_crossbar" # or from the "register_cluster" depends on the control signals. for i in range(num_fu_inports): - s.routing_to_reg_channel[i].recv.msg //= \ + s.register_cluster.recv_data_from_routing_crossbar[i].msg //= \ s.routing_crossbar.send_data[num_tile_outports + i].msg s.register_cluster.write_data_from_routing_crossbar[i] //= \ s.routing_crossbar.send_data[num_tile_outports + i].msg - s.register_cluster.recv_data_from_routing_crossbar[i].msg //= \ - s.routing_to_reg_channel[i].send.msg - s.register_cluster.recv_data_from_routing_crossbar[i].val //= \ - s.routing_to_reg_channel[i].send.val s.register_cluster.recv_data_from_fu_crossbar[i].msg //= \ s.fu_crossbar.send_data[num_tile_outports + i].msg s.register_cluster.recv_data_from_fu_crossbar[i].val //= \ @@ -255,22 +249,21 @@ def update_routing_to_reg_inputs(): is_reg_write = \ s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR - s.routing_to_reg_channel[i].recv.val @= \ + s.register_cluster.recv_data_from_routing_crossbar[i].val @= \ s.routing_crossbar.send_data[num_tile_outports + i].val & \ ~is_reg_write s.register_cluster.write_valid_from_routing_crossbar[i] @= \ s.routing_crossbar.send_data[num_tile_outports + i].val & \ is_reg_write s.routing_crossbar.send_data[num_tile_outports + i].rdy @= \ - is_reg_write | s.routing_to_reg_channel[i].recv.rdy + is_reg_write | \ + s.register_cluster.recv_data_from_routing_crossbar[i].rdy @update def update_reg_cluster_input_rdy(): for i in range(num_tile_outports): s.fu_crossbar.send_data[i].rdy @= s.send_data[i].rdy for i in range(num_fu_inports): - s.routing_to_reg_channel[i].send.rdy @= \ - s.register_cluster.recv_data_from_routing_crossbar[i].rdy s.fu_crossbar.send_data[num_tile_outports + i].rdy @= \ s.register_cluster.recv_data_from_fu_crossbar[i].rdy diff --git a/tile/TileWithContextSwitchRTL.py b/tile/TileWithContextSwitchRTL.py index 284fe62f..101705ce 100644 --- a/tile/TileWithContextSwitchRTL.py +++ b/tile/TileWithContextSwitchRTL.py @@ -245,6 +245,10 @@ def construct(s, IntraCgraPktType, for i in range(num_fu_inports): s.routing_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_routing_crossbar[i] + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_valid_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].val s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] diff --git a/tile/TileWithStreamingLoadRTL.py b/tile/TileWithStreamingLoadRTL.py index aeb3f97a..ea794483 100644 --- a/tile/TileWithStreamingLoadRTL.py +++ b/tile/TileWithStreamingLoadRTL.py @@ -208,6 +208,10 @@ def construct(s, IntraCgraPktType, for i in range(num_fu_inports): s.routing_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_routing_crossbar[i] + s.register_cluster.write_data_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.write_valid_from_routing_crossbar[i] //= \ + s.routing_crossbar.send_data[num_tile_outports + i].val s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] From 02a0aa9cf5344015060120c785aa23729fd5e14a Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:18:55 +0800 Subject: [PATCH 24/60] Fix DivRTL translation regression test --- fu/single/translate/DivRTL_test.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/fu/single/translate/DivRTL_test.py b/fu/single/translate/DivRTL_test.py index 5e831f8a..c35a22bb 100644 --- a/fu/single/translate/DivRTL_test.py +++ b/fu/single/translate/DivRTL_test.py @@ -23,16 +23,20 @@ def test_translate_rem_operator(cmdline_opts): dut.set_metadata(VerilogTranslationPass.explicit_module_name, 'DivRTL') dut.set_metadata(VerilogTranslationPass.explicit_file_name, 'DivRTL__pickled.v') + translate_opts = dict(cmdline_opts) + translate_opts["test_verilog"] = "zeros" + try: - config_model_with_cmdline_opts(dut, cmdline_opts, duts=[]) + config_model_with_cmdline_opts(dut, translate_opts, duts=[]) except VerilogImportError as e: # Translation already emitted Verilog before the optional Verilator import. # On machines without Verilator, still inspect the generated RTL. assert 'verilator: not found' in str(e) verilog = Path('DivRTL__pickled.v').read_text() - assert 'div_remainder = dividend % divisor' in verilog + assert 'div_remainder' in verilog assert "if ( divisor != 32'd0 )" in verilog payload_assigns = [line for line in verilog.splitlines() if 'payload =' in line] - assert all('/' not in line for line in payload_assigns) + assert any('div_remainder' in line for line in payload_assigns) + assert all('/' not in line and '%' not in line for line in payload_assigns) From c400954cfcb2b9b57f160cd76b6065703852cf3f Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:08:18 +0800 Subject: [PATCH 25/60] Fix final RET completion handling --- .../test/CgraRTL_fir_2x2_loop_counter_test.py | 2 +- controller/ControllerRTL.py | 48 +++++++++++++------ fu/single/RetRTL.py | 12 +++-- mem/ctrl/CtrlMemDynamicRTL.py | 8 ++-- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/cgra/test/CgraRTL_fir_2x2_loop_counter_test.py b/cgra/test/CgraRTL_fir_2x2_loop_counter_test.py index fd8d7d43..17456387 100644 --- a/cgra/test/CgraRTL_fir_2x2_loop_counter_test.py +++ b/cgra/test/CgraRTL_fir_2x2_loop_counter_test.py @@ -351,7 +351,7 @@ def sim_fir_with_loop_counter(cmdline_opts, mem_access_is_combinational, has_ctr # the stored result should still be correct thanks to the grant predicate. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 10 + 13 kExpectedOutput = 2215 # Corresponding DFG: diff --git a/controller/ControllerRTL.py b/controller/ControllerRTL.py index ef14d480..6543baaf 100644 --- a/controller/ControllerRTL.py +++ b/controller/ControllerRTL.py @@ -177,6 +177,7 @@ def construct(s, s.dma_spm_addr = Wire(DmaSpmAddrType) s.dma_bytes = Wire(DmaBytesType) s.dma_tag = Wire(DmaTagType) + s.has_ret_kernel = Wire(b1) # Connections. # Requests towards others, 1 cycle delay to improve timing. @@ -193,6 +194,20 @@ def construct(s, s.recv_from_cpu_pkt //= s.recv_from_cpu_pkt_queue.recv s.send_to_cpu_pkt //= s.send_to_cpu_pkt_queue.send + @update_ff + def update_has_ret_kernel(): + if s.reset: + s.has_ret_kernel <<= 0 + else: + cpu_payload = s.recv_from_cpu_pkt_queue.send.msg.payload + is_ret_config = (cpu_payload.cmd == CMD_CONFIG) & ( + (cpu_payload.ctrl.operation == OPT_RET) | + (cpu_payload.ctrl.operation == OPT_RET_VOID)) + config_accepted = (s.recv_from_cpu_pkt_queue.send.val & + s.recv_from_cpu_pkt_queue.send.rdy) + if config_accepted & is_ret_config: + s.has_ret_kernel <<= 1 + @update_ff def update_dma_cmd_regs(): if s.reset: @@ -431,20 +446,25 @@ def update_received_msg(): s.send_to_tile_load_response_queue.recv.val @= 1 elif s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_COMPLETE: - s.recv_from_inter_cgra_noc.rdy @= s.send_to_cpu_pkt_queue.recv.rdy - s.send_to_cpu_pkt_queue.recv.val @= 1 - s.send_to_cpu_pkt_queue.recv.msg @= \ - IntraCgraPktType(s.recv_from_inter_cgra_noc.msg.src_tile_id, # src - s.recv_from_inter_cgra_noc.msg.dst_tile_id, # dst - s.recv_from_inter_cgra_noc.msg.src, # src_cgra_id - s.recv_from_inter_cgra_noc.msg.dst, # src_cgra_id - s.recv_from_inter_cgra_noc.msg.src_x, # src_cgra_x - s.recv_from_inter_cgra_noc.msg.src_y, # src_cgra_y - s.recv_from_inter_cgra_noc.msg.dst_x, # dst_cgra_x - s.recv_from_inter_cgra_noc.msg.dst_y, # dst_cgra_y - 0, # opaque - 0, # vc_id - s.recv_from_inter_cgra_noc.msg.payload) + is_empty_ret_complete = (s.has_ret_kernel & + ~s.recv_from_inter_cgra_noc.msg.payload.data.predicate) + if is_empty_ret_complete: + s.recv_from_inter_cgra_noc.rdy @= 1 + else: + s.recv_from_inter_cgra_noc.rdy @= s.send_to_cpu_pkt_queue.recv.rdy + s.send_to_cpu_pkt_queue.recv.val @= 1 + s.send_to_cpu_pkt_queue.recv.msg @= IntraCgraPktType( + s.recv_from_inter_cgra_noc.msg.src_tile_id, # src + s.recv_from_inter_cgra_noc.msg.dst_tile_id, # dst + s.recv_from_inter_cgra_noc.msg.src, # src_cgra_id + s.recv_from_inter_cgra_noc.msg.dst, # src_cgra_id + s.recv_from_inter_cgra_noc.msg.src_x, # src_cgra_x + s.recv_from_inter_cgra_noc.msg.src_y, # src_cgra_y + s.recv_from_inter_cgra_noc.msg.dst_x, # dst_cgra_x + s.recv_from_inter_cgra_noc.msg.dst_y, # dst_cgra_y + 0, # opaque + 0, # vc_id + s.recv_from_inter_cgra_noc.msg.payload) # Consume and discard the leaf counter complete signal (loop termination # notification from LoopCounter FU) to avoid blocking the NoC. diff --git a/fu/single/RetRTL.py b/fu/single/RetRTL.py index 2df73913..97e31b1a 100644 --- a/fu/single/RetRTL.py +++ b/fu/single/RetRTL.py @@ -73,11 +73,15 @@ def comb_logic(): if s.already_done[s.ctrl_addr_inport]: s.recv_in[s.in0_idx].rdy @= s.recv_all_val s.recv_opt.rdy @= s.recv_all_val - elif s.recv_in[s.in0_idx].msg.predicate & s.recv_opt.msg.is_last_ctrl: + elif s.recv_opt.msg.is_last_ctrl: # Only the final dynamic RET sends the value back to CPU. s.send_to_ctrl_mem.val @= s.recv_all_val & s.reached_vector_factor - # s.send_to_ctrl_mem.msg @= s.recv_in[s.in0_idx].msg - s.send_to_ctrl_mem.msg @= s.CgraPayloadType(CMD_COMPLETE, s.recv_in[s.in0_idx].msg, 0, s.recv_opt.msg, 0) + s.send_to_ctrl_mem.msg @= s.CgraPayloadType( + CMD_COMPLETE, + s.DataType(s.recv_in[s.in0_idx].msg.payload, 1, + s.recv_in[s.in0_idx].msg.bypass, + s.recv_in[s.in0_idx].msg.delay), + 0, s.recv_opt.msg, 0) s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy s.recv_opt.rdy @= s.recv_all_val & s.reached_vector_factor & s.send_to_ctrl_mem.rdy else: @@ -90,7 +94,7 @@ def comb_logic(): if s.already_done[s.ctrl_addr_inport]: s.recv_in[s.in0_idx].rdy @= s.recv_all_val s.recv_opt.rdy @= s.recv_all_val - elif s.recv_in[s.in0_idx].msg.predicate & s.recv_opt.msg.is_last_ctrl: + elif s.recv_opt.msg.is_last_ctrl: # RET_VOID: only the final dynamic RET_VOID notifies ctrl mem. s.send_to_ctrl_mem.val @= s.recv_all_val & s.reached_vector_factor # Sends 0 as data (controller is supposed to know it's RET_VOID based on the operation and data type). diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index a24f3192..a4c73a5f 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -208,8 +208,10 @@ def update_send_ctrl_val(): if s.start_iterate_ctrl == b1(1): if s.sent_complete: s.send_ctrl.val @= 0 - elif ((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ - (s.reg_file.rdata[0].operation == OPT_START): + elif (((s.total_ctrl_steps_val > 0) & \ + (s.times == s.total_ctrl_steps_val) & \ + ~s.has_ret_ctrl) | \ + (s.reg_file.rdata[0].operation == OPT_START)): s.send_ctrl.val @= b1(0) else: s.send_ctrl.val @= 1 @@ -235,7 +237,7 @@ def update_send_ctrl_msg(): s.send_ctrl.msg.is_last_ctrl @= \ s.reg_file.rdata[0].is_last_ctrl | \ ((s.total_ctrl_steps_val > 0) & \ - (s.times == s.total_ctrl_steps_val - TimeType(1))) + (s.times >= s.total_ctrl_steps_val - TimeType(1))) # Keep FUs idle while their prologue inputs are still being skipped. if s.prologue_count_outport_fu != 0: s.send_ctrl.msg.operation @= OPT_NAH From 795c446ae6166344cd3a450b97dfbdc6f8c3338f Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:24:58 +0800 Subject: [PATCH 26/60] Preserve Crossbar done ready bypass --- noc/CrossbarRTL.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index 8c9cc539..d225ce8f 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -250,15 +250,13 @@ def update_in_dir_vector(): def update_rdy_vector(): s.send_rdy_vector @= 0 for i in range(num_outports): - # The `num_inports` indicates the number of outports that go to other tiles. - # Specifically, if the compute already done, we shouldn't care the ones - # (i.e., i >= num_inports) go to the FU's inports. In other words, we skip - # the rdy checking on the FU's inports (connecting from crossbar_outport) if - # the compute is already completed. - if s.in_dir[i] > 0: - # When prologue is active for this output's input, don't - # require the downstream channel to be ready -- we won't be - # sending any data through it during prologue anyway. + # The `outport_towards_local_base_id` indicates the number of outports + # that go to other tiles. Once the FU is done, local FU-input outports + # no longer participate in backpressure for this control step. + if (s.in_dir[i] > 0) & \ + (~s.compute_done | (i < outport_towards_local_base_id)): + # When prologue is active for this output's input, don't require the + # downstream channel to be ready -- we won't send data through it. s.send_rdy_vector[i] @= s.send_data[i].rdy | \ s.prologue_allowing_vector[i] else: From ccd91efeea46e86e6696fdd966cc7203034ed901 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 01:22:42 +0800 Subject: [PATCH 27/60] Align assert test with final RET semantics --- cgra/test/CgraVerifAssert_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cgra/test/CgraVerifAssert_test.py b/cgra/test/CgraVerifAssert_test.py index d0a1ea63..a7f763a4 100644 --- a/cgra/test/CgraVerifAssert_test.py +++ b/cgra/test/CgraVerifAssert_test.py @@ -302,7 +302,7 @@ def line_trace(s): // response should be data_mem[0:4] = [0,1,1,0,1] -// return value should be i (i = 4 in the end) +// return value should be the final loop index (i = 5 at exit) ''' def sim_fir_return(cmdline_opts, mem_access_is_combinational, has_ctrl_ring): @@ -320,12 +320,13 @@ def sim_fir_return(cmdline_opts, mem_access_is_combinational, has_ctrl_ring): kCtrlCountPerIter = 5 kCmpOne = 1 kCmpZero = 0 - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Leave a small drain window after the loop so the final RET and memory + # assertions can complete without delaying return by hundreds of cycles. + # With final-RET semantics, the returned value is the loop-exit index. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 1000 - kExpectedOutput = 4 + 3 + kExpectedOutput = 5 # More details are shown in: # https://github.com/tancheng/VectorCGRA/tree/master/doc/figures/assert_test/DFG.png. From d4ae6856cb9b80635d95cf27a33c8961dc993591 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 02:23:30 +0800 Subject: [PATCH 28/60] Bound context-switch RET test drain window --- cgra/test/CgraWithContextSwitchRTL_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cgra/test/CgraWithContextSwitchRTL_test.py b/cgra/test/CgraWithContextSwitchRTL_test.py index 8da4b85d..84a6fc6e 100644 --- a/cgra/test/CgraWithContextSwitchRTL_test.py +++ b/cgra/test/CgraWithContextSwitchRTL_test.py @@ -346,14 +346,15 @@ def sim_fir_return_two_tasks(cmdline_opts, mem_access_is_combinational): kLoopUpperBound_Task2 = 9 kCtrlCountPerIter_Task1 = 4 kCtrlCountPerIter_Task2 = 3 - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Leave a bounded drain window after each task so final RET observes the + # completed result without delaying context-switch return by hundreds of + # cycles. kTotalCtrlSteps_Task1 = kCtrlCountPerIter_Task1 * \ (kLoopUpperBound_Task1 - kLoopLowerBound) + \ - 100 + 13 kTotalCtrlSteps_Task2 = kCtrlCountPerIter_Task2 * \ (kLoopUpperBound_Task2 - kLoopLowerBound) + \ - 100 + 13 kExpectedOutput_Task1 = 2215 kExpectedOutput_Task2 = 1816 src_opt_pkt = [ From 914b15f6a346523f4a62b691db6763d204f60577 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 07:42:49 +0800 Subject: [PATCH 29/60] Fix kernel return follow-up regressions --- fu/vector/VectorAllReduceRTL.py | 2 +- mem/ctrl/CtrlMemDynamicRTL.py | 12 +++++--- multi_cgra/test/MeshMultiCgraRTL_test.py | 30 +++++++++++-------- .../test/MultiCgraRTL_migration_test.py | 17 +++++++---- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/fu/vector/VectorAllReduceRTL.py b/fu/vector/VectorAllReduceRTL.py index e92f66bb..9d0088c8 100644 --- a/fu/vector/VectorAllReduceRTL.py +++ b/fu/vector/VectorAllReduceRTL.py @@ -137,7 +137,7 @@ def update_signal(): s.recv_in[1].rdy @= (((s.recv_opt.msg.operation == OPT_VEC_REDUCE_ADD_BASE) | \ (s.recv_opt.msg.operation == OPT_VEC_REDUCE_MUL_BASE)) & \ s.send_out[0].rdy) | \ - (((s.recv_opt.msg.operation == OPT_VEC_REDUCE_MUL_BASE_GLOBAL) | \ + (((s.recv_opt.msg.operation == OPT_VEC_REDUCE_ADD_BASE_GLOBAL) | \ (s.recv_opt.msg.operation == OPT_VEC_REDUCE_MUL_BASE_GLOBAL)) & \ s.send_to_ctrl_mem.rdy) s.send_out[0].val @= (s.recv_in[0].val & \ diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index a4c73a5f..e1f37b1a 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -181,24 +181,28 @@ def update_send_pkt_to_controller(): s.send_pkt_to_controller.msg @= IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) s.recv_from_element_queue.send.rdy @= 0 if s.start_iterate_ctrl == b1(1): + element_cmd = s.recv_from_element_queue.send.msg.cmd is_active_ret = s.recv_from_element_queue.send.val & \ ((s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET) | \ (s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET_VOID)) & \ s.recv_from_element_queue.send.msg.data.predicate - if is_active_ret & (~s.sent_complete): + is_ctrl_side_element_msg = s.recv_from_element_queue.send.val & \ + ((element_cmd == CMD_GLOBAL_REDUCE_ADD) | \ + (element_cmd == CMD_GLOBAL_REDUCE_MUL)) + if (is_active_ret | is_ctrl_side_element_msg) & (~s.sent_complete): s.send_pkt_to_controller.msg @= \ - IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, + IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, s.recv_from_element_queue.send.msg) s.send_pkt_to_controller.val @= 1 s.recv_from_element_queue.send.rdy @= s.send_pkt_to_controller.rdy elif s.recv_from_element_queue.send.val: - # Non-RET or predicated-off element responses are not kernel returns. + # Non-RET/predicated-off element responses are not kernel returns. s.recv_from_element_queue.send.rdy @= 1 elif (((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ (s.reg_file.rdata[0].operation == OPT_START)) & ~s.has_ret_ctrl: if ~s.sent_complete: s.send_pkt_to_controller.msg @= \ - IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, + IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) s.send_pkt_to_controller.val @= 1 diff --git a/multi_cgra/test/MeshMultiCgraRTL_test.py b/multi_cgra/test/MeshMultiCgraRTL_test.py index be9d75be..247f32f7 100644 --- a/multi_cgra/test/MeshMultiCgraRTL_test.py +++ b/multi_cgra/test/MeshMultiCgraRTL_test.py @@ -1348,11 +1348,12 @@ def initialize_test_harness(cmdline_opts, kLoopUpperBound = 10 kCtrlCountPerIter = 4 ctrl_steps_per_iter = kCtrlCountPerIter - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Keep a bounded drain window after the final loop iteration. The RET + # complete is now tied to the final control step rather than an early + # predicate-only value. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 100 + 13 ctrl_steps_total = kTotalCtrlSteps kExpectedOutput = 2215 @@ -1922,11 +1923,12 @@ def initialize_test_harness(cmdline_opts, kLoopUpperBound = 10 kCtrlCountPerIter = 4 ctrl_steps_per_iter = kCtrlCountPerIter - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Keep a bounded drain window after the final loop iteration. The RET + # complete is now tied to the final control step rather than an early + # predicate-only value. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 100 + 13 ctrl_steps_total = kTotalCtrlSteps kExpectedOutput = 2215 @@ -2524,11 +2526,12 @@ def initialize_test_harness(cmdline_opts, kLoopUpperBound = 4 kCtrlCountPerIter = 4 ctrl_steps_per_iter = kCtrlCountPerIter - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Keep a bounded drain window after the final loop iteration. The RET + # complete is now tied to the final control step rather than an early + # predicate-only value. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 30 + 13 ctrl_steps_total = kTotalCtrlSteps kExpectedOutput = 2215 @@ -3198,11 +3201,12 @@ def initialize_test_harness(cmdline_opts, kLoopUpperBound = 4 kCtrlCountPerIter = 4 ctrl_steps_per_iter = kCtrlCountPerIter - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Keep a bounded drain window after the final loop iteration. The RET + # complete is now tied to the final control step rather than an early + # predicate-only value. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 30 + 13 ctrl_steps_total = kTotalCtrlSteps kExpectedOutput = 2212 * 2 + kSumInitValue @@ -4375,7 +4379,7 @@ def test_multi_CGRA_fir_vector_global_reduce(cmdline_opts): ['UNSIGNED', 'UNOPTFLAT', 'WIDTH', 'WIDTHCONCAT', 'ALWCOMBORDER']) th = config_model_with_cmdline_opts(th, cmdline_opts, duts = ['dut']) - run_sim(th) + run_sim(th, 300) def test_multi_CGRA_fir_vector_global_reduce_translation(cmdline_opts): th = initialize_test_harness(cmdline_opts, diff --git a/multi_cgra/test/MultiCgraRTL_migration_test.py b/multi_cgra/test/MultiCgraRTL_migration_test.py index f0d71e0f..f03d3350 100644 --- a/multi_cgra/test/MultiCgraRTL_migration_test.py +++ b/multi_cgra/test/MultiCgraRTL_migration_test.py @@ -307,14 +307,19 @@ def initialize_test_harness(cmdline_opts, kCtrlCountPerIter = 3 kCtrlCountPerIter_migration = 2 ctrl_steps_per_iter = kCtrlCountPerIter - # Though kTotalCtrlSteps is way more than required loop iteration count, - # the stored result should still be correct thanks to the grant predicate. + # Keep a bounded drain window after the final loop iteration. The RET + # complete is now tied to the final control step rather than an early + # predicate-only value. kTotalCtrlSteps = kCtrlCountPerIter * \ (kLoopUpperBound - kLoopLowerBound) + \ - 100 + 13 kTotalCtrlSteps_migration = kCtrlCountPerIter_migration * \ (kLoopUpperBound - kLoopLowerBound) + \ - 100 + 13 + # The migrated RET tile issues one RET control per arriving granted value, + # not the full local CGRA schedule. It sees the loop trip count plus the + # remote pipeline drain. + kRetTotalCtrlSteps = (kLoopUpperBound - kLoopLowerBound) + 2 ctrl_steps_total = kTotalCtrlSteps kExpectedOutput = 2215 support_task_switching = False @@ -1027,7 +1032,7 @@ def initialize_test_harness(cmdline_opts, IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_COUNT_PER_ITER, data = DataType(1, 1))), # Pre-configure per-tile total config count. - IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_TOTAL_CTRL_COUNT, data = DataType(kTotalCtrlSteps, 1))), + IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_TOTAL_CTRL_COUNT, data = DataType(kRetTotalCtrlSteps, 1))), # RET. IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, @@ -1686,7 +1691,7 @@ def initialize_test_harness(cmdline_opts, IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_COUNT_PER_ITER, data = DataType(1, 1))), # Pre-configure per-tile total config count. - IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_TOTAL_CTRL_COUNT, data = DataType(kTotalCtrlSteps_migration, 1))), + IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_CONFIG_TOTAL_CTRL_COUNT, data = DataType(kRetTotalCtrlSteps, 1))), # Launch the tile. IntraCgraPktType(0, 1, 0, cgra_2_id, 0, 0, cgra_2_x, cgra_2_y, payload = CgraPayloadType(CMD_LAUNCH)) ], From 13d5c0d7b60982c81297c047ba80091e9be8cf56 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 12:10:30 +0800 Subject: [PATCH 30/60] Fix Verilator latch warnings --- controller/ControllerRTL.py | 1 + mem/ctrl/CtrlMemDynamicRTL.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/controller/ControllerRTL.py b/controller/ControllerRTL.py index 6543baaf..a853194f 100644 --- a/controller/ControllerRTL.py +++ b/controller/ControllerRTL.py @@ -404,6 +404,7 @@ def update_received_msg(): # For the load request from NoC. received_pkt = s.recv_from_inter_cgra_noc.msg + is_empty_ret_complete = b1(0) if s.recv_from_inter_cgra_noc.val: if s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_LOAD_REQUEST: s.send_to_mem_load_request_queue.recv.val @= 1 diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index e1f37b1a..a25421fe 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -180,8 +180,10 @@ def update_send_pkt_to_controller(): s.send_pkt_to_controller.val @= 0 s.send_pkt_to_controller.msg @= IntraCgraPktType(0, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0)) s.recv_from_element_queue.send.rdy @= 0 + element_cmd = s.recv_from_element_queue.send.msg.cmd + is_active_ret = b1(0) + is_ctrl_side_element_msg = b1(0) if s.start_iterate_ctrl == b1(1): - element_cmd = s.recv_from_element_queue.send.msg.cmd is_active_ret = s.recv_from_element_queue.send.val & \ ((s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET) | \ (s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET_VOID)) & \ From 4e4c264d31fdf4e7a18a3c3faee41bc8f63e3323 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:14:01 +0800 Subject: [PATCH 31/60] Generalize routing write bypass --- mem/register_cluster/RegisterClusterRTL.py | 19 ++++++++++--------- .../test/RegisterClusterRTL_test.py | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index ea010256..976b0b97 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -80,17 +80,18 @@ def update_msgs_signals(): # Checks if data should go towards routing_xbar (2 or 3) reg_towards_routing_xbar = active_ctrl & \ ((read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth)) - ret_last_routing_write_bypass = active_ctrl & \ - s.inport_opt.is_last_ctrl & \ - (s.inport_opt.operation == OPT_RET) & \ + routing_write_to_fu_bypass = active_ctrl & \ reg_towards_fu & \ (s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ (s.inport_opt.write_reg_idx[i] == s.inport_opt.read_reg_idx[i]) & \ - s.write_valid_from_routing_crossbar[i] - - # Data from register bank has priority over routing crossbar data for FU path, - # except for final RET reading the same value being written by routing xbar. - if ret_last_routing_write_bypass: + s.write_valid_from_routing_crossbar[i] & \ + ((s.inport_opt.operation != OPT_RET) | \ + s.write_data_from_routing_crossbar[i].predicate) + + # Same-slot routing write/read should be visible to the FU immediately. + # For RET, keep predicated-off routing writes from hiding the register's + # last valid value. + if routing_write_to_fu_bypass: s.send_data_to_fu[i].msg @= s.write_data_from_routing_crossbar[i] elif s.reg_bank[i].send_data.val & reg_towards_fu: s.send_data_to_fu[i].msg @= \ @@ -100,7 +101,7 @@ def update_msgs_signals(): s.recv_data_from_routing_crossbar[i].msg s.send_data_to_fu[i].val @= active_ctrl & \ - (ret_last_routing_write_bypass | \ + (routing_write_to_fu_bypass | \ s.recv_data_from_routing_crossbar[i].val | \ (s.reg_bank[i].send_data.val & reg_towards_fu)) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy diff --git a/mem/register_cluster/test/RegisterClusterRTL_test.py b/mem/register_cluster/test/RegisterClusterRTL_test.py index 9335232b..332b2fbf 100644 --- a/mem/register_cluster/test/RegisterClusterRTL_test.py +++ b/mem/register_cluster/test/RegisterClusterRTL_test.py @@ -347,9 +347,9 @@ def test_reg_cluster_read_towards_both(): src_data_from_const = [[] for _ in range(num_reg_banks)] # Bank 2: reg data (55) goes to both FU and routing_xbar. - # TestSrcRTL starts sending in cycle 2, write lands at end of cycle 2, - # value readable in cycle 3 — two leading DataType(0,0) on both paths. - sink_msgs_fu = [[], [], [DataType(0, 0), DataType(0, 0), DataType(55, 1)], []] + # Same-slot routing-write/read bypass lets the FU observe the value one + # cycle before the registered routing-xbar read path. + sink_msgs_fu = [[], [], [DataType(0, 0), DataType(55, 1)], []] sink_msgs_xbar = [[], [], [DataType(0, 0), DataType(0, 0), DataType(55, 1)], []] th = TestHarnessWithXbarSink( From 6549d3b47c52fc5adb29c68b67245944f96331f3 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:15:31 +0800 Subject: [PATCH 32/60] Clean conv validation debug output --- cgra/test/CgraRTL_conv4x4_test_from_yaml.py | 45 ++++----------------- 1 file changed, 7 insertions(+), 38 deletions(-) diff --git a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py index a712014e..f80b229c 100644 --- a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py +++ b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py @@ -668,7 +668,7 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): debug_progress_enabled = os.environ.get("CGRA_DEBUG_PROGRESS", "0") == "1" debug_every = int(os.environ.get("CGRA_DEBUG_EVERY", "1000")) debug_from_cycle = int(os.environ.get("CGRA_DEBUG_FROM_CYCLE", "0")) - heartbeat_enabled = os.environ.get("CGRA_HEARTBEAT", "1") != "0" + heartbeat_enabled = os.environ.get("CGRA_HEARTBEAT", "0") == "1" debug_tile_ids = [2, 3, 5, 6, 7, 9, 10, 11] debug_elem_tile_ids = [ int(x) for x in os.environ.get("CGRA_DEBUG_ELEM_TILES", @@ -712,36 +712,12 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): "delay", int(cpu_pkt.payload.data.delay), flush=True) if int(cpu_pkt.payload.data.payload) != expected_result: - if hasattr(th.dut, "debug_tile_times"): - print("[mismatch_tile6]", - "times", int(th.dut.debug_tile_times[6]), - "addr", int(th.dut.debug_tile_ctrl_addr[6]), - "op", int(th.dut.debug_tile_op[6]), - "reg0_b0", int(th.dut.debug_tile_reg0_data[6][0]), - "reg0_b0p", int(th.dut.debug_tile_reg0_pred[6][0]), - "reg0_b1", int(th.dut.debug_tile_reg0_data[6][1]), - "reg0_b1p", int(th.dut.debug_tile_reg0_pred[6][1]), - "reg_rd_b0", int(th.dut.debug_tile_reg_read_data[6][0]), - "reg_rd_b0p", int(th.dut.debug_tile_reg_read_pred[6][0]), - "reg_rd_b1", int(th.dut.debug_tile_reg_read_data[6][1]), - "reg_rd_b1p", int(th.dut.debug_tile_reg_read_pred[6][1]), - flush=True) - else: - t6 = th.dut.tile[6] - cm = t6.ctrl_mem - print("[mismatch_tile6]", - "raddr", int(cm.reg_file.raddr[0]), - "times", int(cm.times), - "op", int(cm.send_ctrl.msg.operation), - "reg0_b0", int(t6.register_cluster.debug_reg0[0].payload), - "reg0_b0p", int(t6.register_cluster.debug_reg0[0].predicate), - "reg0_b1", int(t6.register_cluster.debug_reg0[1].payload), - "reg0_b1p", int(t6.register_cluster.debug_reg0[1].predicate), - "reg_rd_b0", int(t6.register_cluster.debug_reg_read[0].payload), - "reg_rd_b0p", int(t6.register_cluster.debug_reg_read[0].predicate), - "reg_rd_b1", int(t6.register_cluster.debug_reg_read[1].payload), - "reg_rd_b1p", int(t6.register_cluster.debug_reg_read[1].predicate), - flush=True) + print("[mismatch_return]", + "cycle", cycle, + "expected", expected_result, + "actual", int(cpu_pkt.payload.data.payload), + "predicate", int(cpu_pkt.payload.data.predicate), + flush=True) if os.environ.get("CGRA_SKIP_BAD_RETURNS", "0") != "1" or \ int(cpu_pkt.payload.data.payload) == expected_result: assert int(cpu_pkt.payload.data.payload) == expected_result @@ -876,13 +852,6 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): f":{int(th.dut.debug_tile_reg_write_data[tid][i])}" f".{int(th.dut.debug_tile_reg_write_pred[tid][i])}" ) - if hasattr(th.dut, "debug_tile_reg0_data"): - reg_parts.append( - f"b{i}rd:{int(th.dut.debug_tile_reg_read_data[tid][i])}" - f".{int(th.dut.debug_tile_reg_read_pred[tid][i])}" - f"$0:{int(th.dut.debug_tile_reg0_data[tid][i])}" - f".{int(th.dut.debug_tile_reg0_pred[tid][i])}" - ) route_parts.append( f"t{tid}:rin[{','.join(recv_parts)}]" f"rout[{','.join(send_parts)}]" From 57db47c98079885d00300dfcc94d253f321aa4f6 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:24:58 +0800 Subject: [PATCH 33/60] Reset GEP stride with default data --- fu/single/GepRTL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fu/single/GepRTL.py b/fu/single/GepRTL.py index 204dbeb5..bf30be76 100644 --- a/fu/single/GepRTL.py +++ b/fu/single/GepRTL.py @@ -159,7 +159,7 @@ def comb_logic(): @update_ff def update_stride(): if s.reset: - s.stride <<= s.DataType(0, 0) + s.stride <<= s.DataType() else: if s.recv_from_ctrl_mem.val & \ (s.recv_from_ctrl_mem.msg.cmd == CMD_CONFIG_GEP_STRIDE): From c2bd82df194b9f8f9698bd85ae802689b8a2d6f8 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:29:25 +0800 Subject: [PATCH 34/60] Use default Verilator import directory --- cgra/test/CgraRTL_conv4x4_test_from_yaml.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py index f80b229c..ebe02413 100644 --- a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py +++ b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py @@ -622,10 +622,6 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): th.dut.set_metadata(VerilogVerilatorImportPass.vl_Wno_list, ['UNSIGNED', 'UNOPTFLAT', 'WIDTH', 'WIDTHCONCAT', 'ALWCOMBORDER']) - th.dut.set_metadata( - VerilogVerilatorImportPass.vl_mk_dir, - os.environ.get("CGRA_VERILATOR_MK_DIR", "obj_dir_conv4x4_light"), - ) verilator_opts = dict(cmdline_opts) verilator_opts["test_verilog"] = "zeros" th = config_model_with_cmdline_opts(th, verilator_opts, duts = ['dut']) From bd7abd7438fceefb8ba0dd1e7385df993db0487b Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:39:24 +0800 Subject: [PATCH 35/60] Allow SUB with predicated zero operand --- fu/single/AdderRTL.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/fu/single/AdderRTL.py b/fu/single/AdderRTL.py index 58dcb180..3d78d5fe 100644 --- a/fu/single/AdderRTL.py +++ b/fu/single/AdderRTL.py @@ -108,9 +108,18 @@ def comb_logic(): elif s.recv_opt.msg.operation == OPT_SUB: s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload - s.recv_in[s.in1_idx].msg.payload - s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \ - s.recv_in[s.in1_idx].msg.predicate & \ - s.reached_vector_factor + if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate: + s.send_out[0].msg.predicate @= s.reached_vector_factor + elif s.recv_in[s.in0_idx].msg.predicate & \ + ~s.recv_in[s.in1_idx].msg.predicate & \ + (s.recv_in[s.in1_idx].msg.payload == s.const_zero.payload): + s.send_out[0].msg.predicate @= s.reached_vector_factor + elif s.recv_in[s.in1_idx].msg.predicate & \ + ~s.recv_in[s.in0_idx].msg.predicate & \ + (s.recv_in[s.in0_idx].msg.payload == s.const_zero.payload): + s.send_out[0].msg.predicate @= s.reached_vector_factor + else: + s.send_out[0].msg.predicate @= 0 s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val s.send_out[0].val @= s.recv_all_val s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy From 6254c7e65cc186751f242051b7f95d6d7dd159d4 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:39:27 +0800 Subject: [PATCH 36/60] Preserve active local crossbar outputs --- noc/CrossbarRTL.py | 94 +++++++++++++++------------------------------- tile/TileRTL.py | 25 +++++++++++- 2 files changed, 55 insertions(+), 64 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index d225ce8f..338cb4f0 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -45,6 +45,7 @@ def construct(s, s.recv_data_val[i] //= s.recv_data[i].val s.crossbar_outport = [InPort(InType) for _ in range(num_outports)] + s.preserve_outport = [InPort(b1) for _ in range(num_outports)] s.send_data = [SendIfcRTL(DataType) for _ in range(num_outports)] s.in_dir = [Wire(InType) for _ in range(num_outports)] @@ -58,7 +59,6 @@ def construct(s, s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) s.crossbar_id = InPort(b1) s.compute_done = InPort(b1) - s.drain_when_inactive = InPort(b1) s.ctrl_addr_inport = InPort(CtrlAddrType) @@ -90,8 +90,8 @@ def construct(s, # Prologue-related wires and registers, which are used to indicate # whether the prologue steps have already been satisfied. - s.prologue_allowing_vector = Wire(num_outports) - s.recv_valid_or_prologue_allowing_vector = Wire(num_outports) + s.during_prologue_allowing_vector = Wire(num_outports) + s.recv_valid_or_during_prologue_allowing_vector = Wire(num_outports) s.prologue_counter = [[Wire(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] s.prologue_counter_next = [[Wire(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] s.prologue_count_inport = [[InPort(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] @@ -126,9 +126,7 @@ def update_signal(): s.all_send_accepted @= 0 for i in range(num_inports): - # Prologued inputs are not consumed; they are ignored for this - # control step and remain available for a later non-prologue step. - s.recv_data[i].rdy @= reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ + s.recv_data[i].rdy @= reduce_and(s.recv_valid_vector) & \ s.all_send_accepted & \ s.recv_required_vector[i] @@ -137,32 +135,17 @@ def update_signal(): # yet accepted in a previous cycle. This prevents duplicate # delivery without making val depend on rdy (send_accepted is # a register, so this is purely val-depends-on-registered-state). - # - # Uses recv_valid_or_prologue_allowing_vector as the gate so - # that prologued inputs (bypassed) do not suppress sends on - # non-prologued outputs. send_required_vector already - # excludes prologued outputs, so they will never assert val. - s.send_data[i].val @= reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ + s.send_data[i].val @= reduce_and(s.recv_valid_vector) & \ s.send_required_vector[i] & \ ~s.send_accepted[i] - if reduce_and(s.recv_valid_or_prologue_allowing_vector) & \ + if reduce_and(s.recv_valid_vector) & \ s.send_required_vector[i] & \ ~s.send_accepted[i]: s.send_data[i].msg.payload @= s.recv_data_msg[s.in_dir_local[i]].payload s.send_data[i].msg.predicate @= s.recv_data_msg[s.in_dir_local[i]].predicate s.recv_opt.rdy @= s.all_send_accepted & \ - reduce_and(s.recv_valid_or_prologue_allowing_vector) - else: - # Unconfigured / idle tiles must not backpressure the fabric. - # If traffic reaches a crossbar with no active route, treat it - # as a sink so stray messages can drain instead of deadlocking - # upstream producers. When this crossbar has already completed the - # current ctrl step, keep inputs backpressured so next-step tokens - # are not dropped while the tile waits for ctrl_proceed. - if s.drain_when_inactive: - for i in range(num_inports): - s.recv_data[i].rdy @= 1 + reduce_and(s.recv_valid_or_during_prologue_allowing_vector) @update_ff def update_prologue_counter(): @@ -217,22 +200,22 @@ def update_send_accepted_next(): @update def update_prologue_allowing_vector(): - s.prologue_allowing_vector @= 0 + s.during_prologue_allowing_vector @= 0 for i in range(num_outports): if s.in_dir[i] > 0: # Records whether the prologue steps have already been satisfied. - s.prologue_allowing_vector[i] @= \ - s.prologue_counter[s.ctrl_addr_inport][s.in_dir_local[i]] < \ - s.prologue_count_wire[s.ctrl_addr_inport][s.in_dir_local[i]] + s.during_prologue_allowing_vector[i] @= \ + (s.prologue_counter[s.ctrl_addr_inport][s.in_dir_local[i]] < \ + s.prologue_count_wire[s.ctrl_addr_inport][s.in_dir_local[i]]) else: - s.prologue_allowing_vector[i] @= 1 + s.during_prologue_allowing_vector[i] @= 0 @update def update_prologue_or_valid_vector(): - s.recv_valid_or_prologue_allowing_vector @= 0 + s.recv_valid_or_during_prologue_allowing_vector @= 0 for i in range(num_outports): - s.recv_valid_or_prologue_allowing_vector[i] @= \ - s.recv_valid_vector[i] | s.prologue_allowing_vector[i] + s.recv_valid_or_during_prologue_allowing_vector[i] @= \ + s.recv_valid_vector[i] | s.during_prologue_allowing_vector[i] @update def update_in_dir_vector(): @@ -250,15 +233,14 @@ def update_in_dir_vector(): def update_rdy_vector(): s.send_rdy_vector @= 0 for i in range(num_outports): - # The `outport_towards_local_base_id` indicates the number of outports - # that go to other tiles. Once the FU is done, local FU-input outports - # no longer participate in backpressure for this control step. + # The `outport_towards_local_base_id` indicates the number of outports that go to other tiles. + # Specifically, if the compute already done, we shouldn't care the ones + # (i.e., i >= outport_towards_local_base_id) go to the FU's inports. In other words, we skip + # the rdy checking on the FU's inports (connecting from crossbar_outport) if + # the compute is already completed. if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id)): - # When prologue is active for this output's input, don't require the - # downstream channel to be ready -- we won't send data through it. - s.send_rdy_vector[i] @= s.send_data[i].rdy | \ - s.prologue_allowing_vector[i] + (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + s.send_rdy_vector[i] @= s.send_data[i].rdy else: s.send_rdy_vector[i] @= 1 @@ -267,18 +249,8 @@ def update_valid_vector(): s.recv_valid_vector @= 0 for i in range(num_outports): if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id)): - # When prologue is active for this output's input, treat the - # input as not-valid even if data is physically present in the - # channel. This prevents the crossbar from asserting - # send_data.val (and therefore potentially blocking on a full - # downstream channel) during prologue steps. Without this - # masking, stale data that happens to sit in a channel can - # trick the crossbar into attempting a send, and if the - # destination channel is full the all-or-nothing semantics - # cause a deadlock. - s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] & \ - ~s.prologue_allowing_vector[i] + (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] else: s.recv_valid_vector[i] @= 1 @@ -288,11 +260,10 @@ def update_recv_required_vector(): s.recv_required_vector[i] @= 0 for i in range(num_outports): - # Prologued inputs are ignored by this control step: do not consume - # their input token, and let recv_valid_or_prologue_allowing_vector - # allow the control stream to proceed without waiting for the token. - if (s.in_dir[i] > 0) & ~s.prologue_allowing_vector[i]: - s.recv_required_vector[s.in_dir_local[i]] @= 1 + if (s.in_dir[i] > 0) & \ + (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + # Avoids crossbar mistakenly consume data during prologue. + s.recv_required_vector[s.in_dir_local[i]] @= ~s.during_prologue_allowing_vector[i] @update def update_send_required_vector(): @@ -301,12 +272,9 @@ def update_send_required_vector(): s.send_required_vector[i] @= 0 for i in range(num_outports): - # An output is required only when it has a mapped input AND - # that input is NOT being bypassed by the prologue mechanism. - # During prologue, the crossbar must not attempt to send data - # on outputs fed by prologued inputs; otherwise a full - # downstream channel would block the entire crossbar. - if (s.in_dir[i] > 0) & ~s.prologue_allowing_vector[i]: + if (s.in_dir[i] > 0) & \ + ~s.during_prologue_allowing_vector[i] & \ + (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): s.send_required_vector[i] @= 1 diff --git a/tile/TileRTL.py b/tile/TileRTL.py index fcfb2097..13931764 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -31,6 +31,7 @@ from ..noc.CrossbarRTL import CrossbarRTL from ..noc.LinkOrRTL import LinkOrRTL from ..noc.PyOCN.pymtl3_net.channel.ChannelRTL import ChannelRTL +from ..lib.basic.val_rdy.queues import PipeQueueRTL from ..rf.RegisterRTL import RegisterRTL from ..lib.util.data_struct_attr import * @@ -116,7 +117,7 @@ def construct(s, IntraCgraPktType, # The `tile_in_channel` indicates the outport channels that are # connected to the next tiles. - s.tile_in_channel = [ChannelRTL(DataType, latency = 1) + s.tile_in_channel = [ChannelRTL(DataType, QueueType=PipeQueueRTL, latency = 1) for _ in range(num_tile_inports)] # The `tile_out_or_link` would "or" the outports of the @@ -128,6 +129,7 @@ def construct(s, IntraCgraPktType, s.element_done = Wire(1) s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) + s.routing_preserve_local = [Wire(b1) for _ in range(num_fu_inports)] s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -200,6 +202,27 @@ def construct(s, IntraCgraPktType, s.fu_crossbar.crossbar_outport[i] //= \ s.ctrl_mem.send_ctrl.msg.fu_xbar_outport[i] + if i < num_tile_outports: + s.routing_crossbar.preserve_outport[i] //= 0 + s.fu_crossbar.preserve_outport[i] //= 0 + else: + local_idx = i - num_tile_outports + s.routing_crossbar.preserve_outport[i] //= \ + s.routing_preserve_local[local_idx] + s.fu_crossbar.preserve_outport[i] //= 0 + + + @update + def update_routing_preserve_local(): + for i in range(num_fu_inports): + read_towards = s.ctrl_mem.send_ctrl.msg.read_reg_towards[i] + read_towards_routing = \ + (read_towards == READ_TOWARDS_ROUTING_XBAR) | \ + (read_towards == READ_TOWARDS_BOTH) + s.routing_preserve_local[i] @= \ + (s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR) | \ + read_towards_routing + # Connections on the `fu_crossbar`. for i in range(num_fu_outports): s.element.send_out[i] //= s.fu_crossbar.recv_data[i] From f4ba4c8f0942bc875b1272ce38fecd5a023e66fe Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 37/60] Explain RET completion paths --- mem/ctrl/CtrlMemDynamicRTL.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index a25421fe..8c7267a9 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -184,6 +184,9 @@ def update_send_pkt_to_controller(): is_active_ret = b1(0) is_ctrl_side_element_msg = b1(0) if s.start_iterate_ctrl == b1(1): + # Only a real, predicated RET/RET_VOID is allowed to complete a + # dynamic kernel. Other element messages can share this queue, so + # forwarding them as COMPLETE would terminate the kernel early. is_active_ret = s.recv_from_element_queue.send.val & \ ((s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET) | \ (s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET_VOID)) & \ @@ -200,8 +203,14 @@ def update_send_pkt_to_controller(): elif s.recv_from_element_queue.send.val: # Non-RET/predicated-off element responses are not kernel returns. s.recv_from_element_queue.send.rdy @= 1 + # Kernels without RET still need the legacy total-step completion + # path. Example: systolic/debug kernels may never enqueue a RET, so + # the controller must finish once the configured step count retires. elif (((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ (s.reg_file.rdata[0].operation == OPT_START)) & ~s.has_ret_ctrl: + # Keep the legacy timeout COMPLETE only for kernels with no RET in + # their control memory, e.g., systolic/debug kernels that rely on + # total_ctrl_steps_val. RET kernels use the explicit path above. if ~s.sent_complete: s.send_pkt_to_controller.msg @= \ IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, @@ -279,6 +288,8 @@ def issue_complete(): @update_ff def record_ret_ctrl(): + # Once any configured control word is RET/RET_VOID, suppress the + # total_ctrl_steps fallback so the returned data must come from RET. if s.reset: s.has_ret_ctrl <<= 0 elif s.recv_pkt_from_controller_queue.send.val & \ From 8026f4c9139ec3291829a33adfb1a495721cb600 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 38/60] Explain divider remainder implementation --- fu/single/DivRTL.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fu/single/DivRTL.py b/fu/single/DivRTL.py index d81d860e..20fc4275 100644 --- a/fu/single/DivRTL.py +++ b/fu/single/DivRTL.py @@ -41,6 +41,10 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0 s.div_quotient = Wire(PayloadType) s.div_remainder = Wire(PayloadType) + # Compute quotient and remainder with shift/subtract logic instead of + # Python percent/modulo so Verilator translation sees only fixed-width RTL + # operations. Divisor zero is defined as quotient=0 and remainder=0, + # matching the deterministic zero-divisor behavior used by the tests. @update def comb_div_rem(): quotient = PayloadType(0) From 28d570ac268d1072dbfee491f63cb6c5dfa20d20 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 39/60] Explain zero identity predicate --- fu/single/AdderRTL.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fu/single/AdderRTL.py b/fu/single/AdderRTL.py index 3d78d5fe..25bec7cb 100644 --- a/fu/single/AdderRTL.py +++ b/fu/single/AdderRTL.py @@ -67,6 +67,9 @@ def comb_logic(): if s.recv_opt.val: if s.recv_opt.msg.operation == OPT_ADD: s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload + s.recv_in[s.in1_idx].msg.payload + # Treat a predicated-off zero as the additive identity. Example: + # (value=7,pred=1) + (value=0,pred=0) should produce pred=1, + # while (value=7,pred=1) + (value=5,pred=0) stays pred=0. if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate: s.send_out[0].msg.predicate @= s.reached_vector_factor elif s.recv_in[s.in0_idx].msg.predicate & \ @@ -108,6 +111,8 @@ def comb_logic(): elif s.recv_opt.msg.operation == OPT_SUB: s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload - s.recv_in[s.in1_idx].msg.payload + # Same identity rule for SUB: subtracting an inactive zero keeps + # the active operand live; subtracting an inactive nonzero does not. if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate: s.send_out[0].msg.predicate @= s.reached_vector_factor elif s.recv_in[s.in0_idx].msg.predicate & \ From 1d95bc2ccd7e671a7b1b533572eb87f13da343f3 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 40/60] Explain crossbar prologue semantics --- noc/CrossbarRTL.py | 25 +++++++++++++++++-------- tile/TileRTL.py | 3 +++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index 338cb4f0..c13fc56e 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -46,6 +46,9 @@ def construct(s, s.crossbar_outport = [InPort(InType) for _ in range(num_outports)] s.preserve_outport = [InPort(b1) for _ in range(num_outports)] + # Local outputs normally stop participating once the FU is done. + # preserve_outport keeps selected local outputs live when the same control + # step still needs routing-xbar data for a register write or FU operand. s.send_data = [SendIfcRTL(DataType) for _ in range(num_outports)] s.in_dir = [Wire(InType) for _ in range(num_outports)] @@ -88,8 +91,12 @@ def construct(s, # in the current cycle via send_rdy_vector). s.all_send_accepted = Wire(b1) - # Prologue-related wires and registers, which are used to indicate - # whether the prologue steps have already been satisfied. + # Prologue-related wires and registers. A prologued input lets the + # current control step proceed as if that input were present, but the + # token is neither required nor consumed until a later non-prologue step. + # Example: a one-cycle routing prologue on input N skips waiting for N in + # this ctrl step; if token A is already sitting on N, A stays queued for + # the first real routed step. s.during_prologue_allowing_vector = Wire(num_outports) s.recv_valid_or_during_prologue_allowing_vector = Wire(num_outports) s.prologue_counter = [[Wire(PrologueCountType) for _ in range(num_inports)] for _ in range(ctrl_mem_size)] @@ -233,11 +240,10 @@ def update_in_dir_vector(): def update_rdy_vector(): s.send_rdy_vector @= 0 for i in range(num_outports): - # The `outport_towards_local_base_id` indicates the number of outports that go to other tiles. - # Specifically, if the compute already done, we shouldn't care the ones - # (i.e., i >= outport_towards_local_base_id) go to the FU's inports. In other words, we skip - # the rdy checking on the FU's inports (connecting from crossbar_outport) if - # the compute is already completed. + # After the FU finishes, normal local FU-input outports no longer need + # to backpressure this crossbar. preserve_outport is the exception: a + # local routing output still matters when it writes a register or feeds + # an operand selected from the routing crossbar in this same ctrl step. if (s.in_dir[i] > 0) & \ (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): s.send_rdy_vector[i] @= s.send_data[i].rdy @@ -262,7 +268,10 @@ def update_recv_required_vector(): for i in range(num_outports): if (s.in_dir[i] > 0) & \ (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): - # Avoids crossbar mistakenly consume data during prologue. + # During prologue, an input token is intentionally skipped for this + # ctrl step. Example: if out0 uses in1 with prologue=1, control may + # advance without consuming in1; the next non-prologue step will see + # the token still aligned with its real consumer. s.recv_required_vector[s.in_dir_local[i]] @= ~s.during_prologue_allowing_vector[i] @update diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 13931764..3d3f29eb 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -202,6 +202,9 @@ def construct(s, IntraCgraPktType, s.fu_crossbar.crossbar_outport[i] //= \ s.ctrl_mem.send_ctrl.msg.fu_xbar_outport[i] + # Only routing-xbar outputs that target the local register/FU side need + # preservation. Tile-to-tile outputs still follow normal compute_done + # gating because downstream tiles consume them through send_data. if i < num_tile_outports: s.routing_crossbar.preserve_outport[i] //= 0 s.fu_crossbar.preserve_outport[i] //= 0 From 56e4fc15cc449f42e3f5c1690deddf6ff804a43b Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 41/60] Explain routing write bypass --- mem/register_cluster/RegisterClusterRTL.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index 976b0b97..b6df119d 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -80,6 +80,10 @@ def update_msgs_signals(): # Checks if data should go towards routing_xbar (2 or 3) reg_towards_routing_xbar = active_ctrl & \ ((read_towards == kReadTowardsRoutingXbar) | (read_towards == kReadTowardsBoth)) + # Same ctrl slot can both write a routing value into a register and + # read that register for the FU. The register file would expose the old + # value for that cycle, so bypass the routing write when the read/write + # indices match. Example: routing_xbar writes r3=99 while RET reads r3. routing_write_to_fu_bypass = active_ctrl & \ reg_towards_fu & \ (s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ @@ -89,8 +93,10 @@ def update_msgs_signals(): s.write_data_from_routing_crossbar[i].predicate) # Same-slot routing write/read should be visible to the FU immediately. - # For RET, keep predicated-off routing writes from hiding the register's - # last valid value. + # Example: a final RET can write reg3 from the routing xbar and read + # reg3 for the FU in the same control step. Use the routing value only + # when the write is valid; a predicated-off RET write must not hide the + # register bank's last valid value. if routing_write_to_fu_bypass: s.send_data_to_fu[i].msg @= s.write_data_from_routing_crossbar[i] elif s.reg_bank[i].send_data.val & reg_towards_fu: From cd0733285b2bf759fa8f03325c3ea53f85d41756 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 42/60] Explain GEP stride config path --- controller/ControllerRTL.py | 3 +++ fu/single/GepRTL.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/controller/ControllerRTL.py b/controller/ControllerRTL.py index 8e3ebfca..3fdc5ea5 100644 --- a/controller/ControllerRTL.py +++ b/controller/ControllerRTL.py @@ -484,6 +484,9 @@ def update_received_msg(): (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \ (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \ (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) : + # These packets are tile-local configuration, including GEP stride. + # Forward them on the control ring; treating them as controller-local + # commands would leave the target tile running GEP with a stale stride. s.recv_from_inter_cgra_noc.rdy @= s.send_to_ctrl_ring_pkt.rdy s.send_to_ctrl_ring_pkt.val @= s.recv_from_inter_cgra_noc.val s.send_to_ctrl_ring_pkt.msg @= \ diff --git a/fu/single/GepRTL.py b/fu/single/GepRTL.py index bf30be76..8f0b2b15 100644 --- a/fu/single/GepRTL.py +++ b/fu/single/GepRTL.py @@ -159,6 +159,9 @@ def comb_logic(): @update_ff def update_stride(): if s.reset: + # DataType has payload/predicate/bypass/delay fields. Use the default + # constructor so Verilator sees all fields reset, instead of passing + # only payload/predicate. s.stride <<= s.DataType() else: if s.recv_from_ctrl_mem.val & \ From 75de9b3d682c4b6d6cfffdf978ef248a03a8abf0 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 43/60] Explain grant once state --- fu/single/GrantRTL.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fu/single/GrantRTL.py b/fu/single/GrantRTL.py index 7001e32b..a85ca8b6 100644 --- a/fu/single/GrantRTL.py +++ b/fu/single/GrantRTL.py @@ -31,7 +31,8 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0 s.in1_idx = Wire(idx_nbits) s.recv_all_val = Wire(1) # Per-slot (per ctrl_addr) latch so that multiple GRANT_ONCE ops in the - # same tile do not share state. + # same tile do not share state. Example: ctrl slot 2 can grant loop bound + # A once, while ctrl slot 5 independently grants loop bound B once. num_slots = 1 << s.CtrlAddrType.nbits s.already_grt_once = [Wire(1) for _ in range(num_slots)] s.cur_already_grt_once = Wire(1) @@ -115,10 +116,12 @@ def comb_logic(): s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy elif s.recv_opt.msg.operation == OPT_GRT_ONCE_CONST: - # GRANT_ONCE_CONST: every non-prologue execution consumes one entry from - # the const queue, but predicate=1 is emitted only on the first such - # execution for this ctrl_addr; subsequent executions emit predicate=0. - # (Prologue cycles see OPT_NAH via FlexibleFuRTL and do not reach here.) + # GRANT_ONCE_CONST: every real execution consumes the next const so + # the queue remains aligned with the control stream, but only the + # first execution for this ctrl_addr emits predicate=1. Example: two + # different loop-header constants in the same tile use different + # ctrl_addr slots; granting one must not suppress the other. Repeated + # executions of the same slot emit predicates 1, 0, 0, ... s.send_out[0].msg @= s.recv_const.msg s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.cur_already_grt_once From 16bc770fd8a26ce9846e962dc31e3c8aa4cbe7d3 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 44/60] Explain pending const consumes --- mem/const/ConstQueueDynamicRTL.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mem/const/ConstQueueDynamicRTL.py b/mem/const/ConstQueueDynamicRTL.py index 76db513f..c1c3146e 100644 --- a/mem/const/ConstQueueDynamicRTL.py +++ b/mem/const/ConstQueueDynamicRTL.py @@ -101,12 +101,11 @@ def update_rd_cur(): s.rd_cur <<= 0 s.consume_pending <<= 0 else: - # A consumption is "owed" whenever the reader raises rdy, or one - # was already pending from an earlier cycle. It retires on the next - # ctrl_proceed pulse. (Matches the original advance condition when - # rdy and ctrl_proceed happen in the same cycle, but also covers the - # case where rdy was asserted one cycle and ctrl_proceed only pulses - # later; previously that handshake was silently lost.) + # A const read is retired only when both sides of the tile control step + # agree. Example: cycle N has send_const.rdy=1 but routing_xbar keeps + # ctrl_proceed=0; remember that owed consume, then advance rd_cur when + # ctrl_proceed pulses at cycle N+1. Without consume_pending, const[0] + # would be presented again and all later const operands would shift. handshake_now = s.send_const.rdy consume_retire = (s.consume_pending | handshake_now) & s.ctrl_proceed if consume_retire: From 7a3a646d66e98c9e57b2e9c78f7d3c9f8255a78d Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 45/60] Explain tile completion handshakes --- tile/TileRTL.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index cdffb796..5a19c045 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -246,6 +246,11 @@ def update_routing_to_reg_inputs(): for i in range(num_tile_outports): s.routing_crossbar.send_data[i].rdy @= s.send_data[i].rdy for i in range(num_fu_inports): + # A local routing-crossbar output is either an operand for the FU or a + # register write, never both in this interface. Split val/rdy so a + # register write can complete without waiting for the FU input port. + # Example: DATA_MOV writes r2 from routing_xbar; recv_data_to_fu stays + # invalid, write_valid_from_routing_crossbar carries the token. is_reg_write = \ s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR @@ -322,9 +327,10 @@ def update_opt(): # FIXME: Do we still need separate element and routing_xbar? # FIXME: Do we need to consider reg bank here? - # Keep FU-side control live until the FU crossbar has also consumed - # the result. Otherwise a FU that finishes one cycle ahead of the - # fu_crossbar can drop its output while ctrl still waits. + # Keep FU-side control live until the FU crossbar has also consumed the + # result. Example: the element produces a MUL result this cycle, but the + # FU crossbar output is backpressured; advancing ctrl now would let the + # element overwrite/drop that result before the crossbar accepts it. s.element.recv_opt.val @= s.ctrl_mem.send_ctrl.val & \ ~(s.element_done & s.fu_crossbar_done) s.routing_crossbar.recv_opt.val @= s.ctrl_mem.send_ctrl.val & ~s.routing_crossbar_done From 43fcb4f8d8f2b49dd2546cea0e8e118ccfa850df Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:05:02 +0800 Subject: [PATCH 46/60] Explain conv validation choices --- cgra/test/CgraRTL_conv4x4_test_from_yaml.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py index ebe02413..3f5c290c 100644 --- a/cgra/test/CgraRTL_conv4x4_test_from_yaml.py +++ b/cgra/test/CgraRTL_conv4x4_test_from_yaml.py @@ -580,6 +580,10 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): preload_drain_cycles = 0 if use_verilator: + # Verilator runs the full 4x4 conv quickly enough once data is preloaded + # through CPU store packets. Pure-Python mode can directly preload memory + # below so debug runs can inspect internal tile state without spending + # thousands of cycles sending stores. src_ctrl_pkt.extend(make_preload_packets()) preload_pkt_count = 2 * total preload_drain_cycles = int(os.environ.get("CGRA_PRELOAD_DRAIN_CYCLES", "10000")) @@ -622,6 +626,9 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): th.dut.set_metadata(VerilogVerilatorImportPass.vl_Wno_list, ['UNSIGNED', 'UNOPTFLAT', 'WIDTH', 'WIDTHCONCAT', 'ALWCOMBORDER']) + # Keep PyMTL's default Verilator object directory. Overriding vl_mk_dir + # makes the generated C++ wrapper include one directory while Verilator + # emits sources into another, which fails during shared-library build. verilator_opts = dict(cmdline_opts) verilator_opts["test_verilog"] = "zeros" th = config_model_with_cmdline_opts(th, verilator_opts, duts = ['dut']) @@ -696,6 +703,11 @@ def sim_conv(cmdline_opts, mem_access_is_combinational): trace_logger.log_cycle(th.dut) if int(th.dut.send_to_cpu_pkt.val) & int(th.dut.send_to_cpu_pkt.rdy): cpu_pkt = th.dut.send_to_cpu_pkt.msg + # A predicated-off RET can appear on the control path but is not an + # architectural kernel return. Only predicate=1 COMPLETE packets count. + # Some intermediate RET-like packets can be predicated off. The kernel + # completes only on a predicated CMD_COMPLETE carrying the expected value; + # ignoring predicate=0 here keeps the test aligned with CtrlMemDynamicRTL. if int(cpu_pkt.payload.data.predicate) == 0: continue print("cpu_pkt:", From 29d89bae62f07772952307a287ba4ea2956fc62b Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:13:52 +0800 Subject: [PATCH 47/60] Clarify RET completion route --- mem/ctrl/CtrlMemDynamicRTL.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mem/ctrl/CtrlMemDynamicRTL.py b/mem/ctrl/CtrlMemDynamicRTL.py index 8c7267a9..a00b994e 100644 --- a/mem/ctrl/CtrlMemDynamicRTL.py +++ b/mem/ctrl/CtrlMemDynamicRTL.py @@ -185,8 +185,10 @@ def update_send_pkt_to_controller(): is_ctrl_side_element_msg = b1(0) if s.start_iterate_ctrl == b1(1): # Only a real, predicated RET/RET_VOID is allowed to complete a - # dynamic kernel. Other element messages can share this queue, so - # forwarding them as COMPLETE would terminate the kernel early. + # dynamic kernel. The RET payload is wrapped in CMD_COMPLETE and sent + # to the controller, which is the path back to the CPU. Other element + # messages can share this queue, so forwarding them as COMPLETE would + # terminate the kernel early. is_active_ret = s.recv_from_element_queue.send.val & \ ((s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET) | \ (s.recv_from_element_queue.send.msg.ctrl.operation == OPT_RET_VOID)) & \ @@ -208,9 +210,6 @@ def update_send_pkt_to_controller(): # the controller must finish once the configured step count retires. elif (((s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val)) | \ (s.reg_file.rdata[0].operation == OPT_START)) & ~s.has_ret_ctrl: - # Keep the legacy timeout COMPLETE only for kernels with no RET in - # their control memory, e.g., systolic/debug kernels that rely on - # total_ctrl_steps_val. RET kernels use the explicit path above. if ~s.sent_complete: s.send_pkt_to_controller.msg @= \ IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, From cd07f921b75c93113923eb5fb4aa4fc8dc6b47d1 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:15:38 +0800 Subject: [PATCH 48/60] Explain DivRTL translation import failure --- fu/single/translate/DivRTL_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fu/single/translate/DivRTL_test.py b/fu/single/translate/DivRTL_test.py index c35a22bb..99b44282 100644 --- a/fu/single/translate/DivRTL_test.py +++ b/fu/single/translate/DivRTL_test.py @@ -29,8 +29,10 @@ def test_translate_rem_operator(cmdline_opts): try: config_model_with_cmdline_opts(dut, translate_opts, duts=[]) except VerilogImportError as e: - # Translation already emitted Verilog before the optional Verilator import. - # On machines without Verilator, still inspect the generated RTL. + # This is not a PyMTL translation failure: config_model_with_cmdline_opts + # first emits DivRTL__pickled.v, then optionally imports it with Verilator + # when test_verilog is enabled. CI/dev machines without Verilator can fail + # the import step even though the generated RTL is present and checkable. assert 'verilator: not found' in str(e) verilog = Path('DivRTL__pickled.v').read_text() From ca73e316a9a08a57e7c789ad522fa35b7c549f85 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 23:17:19 +0800 Subject: [PATCH 49/60] Clarify zero identity predicate --- fu/single/AdderRTL.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fu/single/AdderRTL.py b/fu/single/AdderRTL.py index 25bec7cb..812cf220 100644 --- a/fu/single/AdderRTL.py +++ b/fu/single/AdderRTL.py @@ -67,9 +67,11 @@ def comb_logic(): if s.recv_opt.val: if s.recv_opt.msg.operation == OPT_ADD: s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload + s.recv_in[s.in1_idx].msg.payload - # Treat a predicated-off zero as the additive identity. Example: - # (value=7,pred=1) + (value=0,pred=0) should produce pred=1, - # while (value=7,pred=1) + (value=5,pred=0) stays pred=0. + # Treat a predicated-off zero as the additive identity: a zero value with + # predicate=0 is a neutral placeholder, not a missing contributing + # operand. Example: (value=7,pred=1) + (value=0,pred=0) should + # produce pred=1, while (value=7,pred=1) + (value=5,pred=0) stays + # pred=0. if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate: s.send_out[0].msg.predicate @= s.reached_vector_factor elif s.recv_in[s.in0_idx].msg.predicate & \ From 783a78348aad7b09529822b30ced834282b82e4a Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 17 Jul 2026 01:15:08 +0800 Subject: [PATCH 50/60] Add inactive crossbar drain port --- noc/CrossbarRTL.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index c13fc56e..e37dd977 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -62,6 +62,7 @@ def construct(s, s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) s.crossbar_id = InPort(b1) s.compute_done = InPort(b1) + s.drain_when_inactive = InPort(b1) s.ctrl_addr_inport = InPort(CtrlAddrType) @@ -153,6 +154,12 @@ def update_signal(): s.recv_opt.rdy @= s.all_send_accepted & \ reduce_and(s.recv_valid_or_during_prologue_allowing_vector) + elif s.drain_when_inactive: + # Test-only/manual drain path: when no routing op is active, consume + # queued input tokens without producing outputs. Tile-level users tie + # this low, so normal scheduled execution is unchanged. + for i in range(num_inports): + s.recv_data[i].rdy @= 1 @update_ff def update_prologue_counter(): From 8e3ffc5d04a1e81eefef27127d0dd2b5e220b7dd Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:17:42 +0800 Subject: [PATCH 51/60] Restore crossbar prologue warm-up handshake --- noc/CrossbarRTL.py | 25 +++++++++---------------- noc/test/CrossbarRTL_test.py | 5 +++-- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index e37dd977..e26330db 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -247,12 +247,10 @@ def update_in_dir_vector(): def update_rdy_vector(): s.send_rdy_vector @= 0 for i in range(num_outports): - # After the FU finishes, normal local FU-input outports no longer need - # to backpressure this crossbar. preserve_outport is the exception: a - # local routing output still matters when it writes a register or feeds - # an operand selected from the routing crossbar in this same ctrl step. + # After the FU finishes, only directions toward a local consumer still + # backpressure the crossbar; routed outputs may drain independently. if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + (~s.compute_done | (i < outport_towards_local_base_id)): s.send_rdy_vector[i] @= s.send_data[i].rdy else: s.send_rdy_vector[i] @= 1 @@ -261,8 +259,7 @@ def update_rdy_vector(): def update_valid_vector(): s.recv_valid_vector @= 0 for i in range(num_outports): - if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + if s.in_dir[i] > 0: s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] else: s.recv_valid_vector[i] @= 1 @@ -273,12 +270,10 @@ def update_recv_required_vector(): s.recv_required_vector[i] @= 0 for i in range(num_outports): - if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): - # During prologue, an input token is intentionally skipped for this - # ctrl step. Example: if out0 uses in1 with prologue=1, control may - # advance without consuming in1; the next non-prologue step will see - # the token still aligned with its real consumer. + if s.in_dir[i] > 0: + # A prologue forwards a warm-up copy without dequeuing this token. + # The following real step consumes it, even after the local FU has + # completed, so it cannot block the upstream tile. s.recv_required_vector[s.in_dir_local[i]] @= ~s.during_prologue_allowing_vector[i] @update @@ -288,9 +283,7 @@ def update_send_required_vector(): s.send_required_vector[i] @= 0 for i in range(num_outports): - if (s.in_dir[i] > 0) & \ - ~s.during_prologue_allowing_vector[i] & \ - (~s.compute_done | (i < outport_towards_local_base_id) | s.preserve_outport[i]): + if s.in_dir[i] > 0: s.send_required_vector[i] @= 1 diff --git a/noc/test/CrossbarRTL_test.py b/noc/test/CrossbarRTL_test.py index dda5f847..314e8ae4 100644 --- a/noc/test/CrossbarRTL_test.py +++ b/noc/test/CrossbarRTL_test.py @@ -160,7 +160,7 @@ def test_multi_cast(): num_routing_outports, src_data, src_opt, sink_out) run_sim(th) -def test_prologue_preserves_skipped_input(): +def test_prologue_warms_and_preserves_input(): src_opt = [CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)]), @@ -168,7 +168,8 @@ def test_prologue_preserves_skipped_input(): [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)])] src_data = [[DataType(7, 1)], [], []] - sink_out = [[DataType(7, 1)], [], []] + # The prologue emits a warm-up copy; the following real step consumes it. + sink_out = [[DataType(7, 1), DataType(7, 1)], [], []] th = TestHarness(FU, DataType, CtrlType, num_tile_inports, num_routing_outports, src_data, src_opt, sink_out, prologue_counts = {(0, 0): 1}) From 8d6a40b3f39db9c8e346be1cf8b3253d55af8dc7 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 23 Jul 2026 02:17:52 +0800 Subject: [PATCH 52/60] Prevent dropped routing tokens at register cluster --- mem/register_cluster/RegisterClusterRTL.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index b6df119d..316a4a6d 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -112,12 +112,11 @@ def update_msgs_signals(): (s.reg_bank[i].send_data.val & reg_towards_fu)) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy + # A ready response consumes the routing token. It is safe only when + # it performs the legacy NAH register write or the FU accepts it. s.recv_data_from_routing_crossbar[i].rdy @= \ - (~active_ctrl | \ - ((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + (((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ (s.inport_opt.operation == OPT_NAH)) | \ - (s.inport_opt.fu_in[i] == 0) | \ - reg_towards_fu | \ s.send_data_to_fu[i].rdy) s.recv_data_from_fu_crossbar[i].rdy @= 1 s.recv_data_from_const[i].rdy @= 1 From 60977aa75ac4fc16775f92c6feb3e375ad024d22 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:15:46 +0800 Subject: [PATCH 53/60] Scope routing write bypass to RET handshakes --- mem/register_cluster/RegisterClusterRTL.py | 19 +++++++++++++------ tile/TileRTL.py | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/mem/register_cluster/RegisterClusterRTL.py b/mem/register_cluster/RegisterClusterRTL.py index 316a4a6d..e212c91f 100644 --- a/mem/register_cluster/RegisterClusterRTL.py +++ b/mem/register_cluster/RegisterClusterRTL.py @@ -41,6 +41,7 @@ def construct(s, DataType, CtrlType, num_reg_banks, s.recv_data_from_const = [RecvIfcRTL(DataType) for _ in range(num_reg_banks)] s.write_data_from_routing_crossbar = [InPort(DataType) for _ in range(num_reg_banks)] s.write_valid_from_routing_crossbar = [InPort(b1) for _ in range(num_reg_banks)] + s.routing_write_valid = [Wire(b1) for _ in range(num_reg_banks)] s.send_data_to_fu = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] # Direct output from register banks towards routing crossbar (bypasses FU). s.send_data_to_routing_crossbar = [SendIfcRTL(DataType) for _ in range(num_reg_banks)] @@ -55,10 +56,16 @@ def construct(s, DataType, CtrlType, num_reg_banks, s.reg_bank[i].inport_wdata[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_data_from_routing_crossbar[i] s.reg_bank[i].inport_wdata[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].msg s.reg_bank[i].inport_wdata[PORT_INDEX_CONST] //= s.recv_data_from_const[i].msg - s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.write_valid_from_routing_crossbar[i] + s.reg_bank[i].inport_valid[PORT_INDEX_ROUTING_CROSSBAR] //= s.routing_write_valid[i] s.reg_bank[i].inport_valid[PORT_INDEX_FU_CROSSBAR] //= s.recv_data_from_fu_crossbar[i].val s.reg_bank[i].inport_valid[PORT_INDEX_CONST] //= s.recv_data_from_const[i].val + @update + def update_routing_write_valid(): + for i in range(num_reg_banks): + s.routing_write_valid[i] @= s.recv_data_from_routing_crossbar[i].val | \ + s.write_valid_from_routing_crossbar[i] + @update def update_msgs_signals(): # Initializes signals. @@ -112,12 +119,12 @@ def update_msgs_signals(): (s.reg_bank[i].send_data.val & reg_towards_fu)) s.reg_bank[i].send_data.rdy @= s.send_data_to_fu[i].rdy - # A ready response consumes the routing token. It is safe only when - # it performs the legacy NAH register write or the FU accepts it. + # A routing token may be retired when this control does not select + # the FU input; this keeps prologue/NAH tokens from shifting the + # following active step. s.recv_data_from_routing_crossbar[i].rdy @= \ - (((s.inport_opt.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ - (s.inport_opt.operation == OPT_NAH)) | \ - s.send_data_to_fu[i].rdy) + (s.inport_opt.operation == OPT_NAH) | \ + (s.inport_opt.fu_in[i] == 0) | s.send_data_to_fu[i].rdy s.recv_data_from_fu_crossbar[i].rdy @= 1 s.recv_data_from_const[i].rdy @= 1 diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 89cb023b..88bda25b 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -24,6 +24,7 @@ from ..lib.basic.val_rdy.ifcs import ValRdyRecvIfcRTL as RecvIfcRTL from ..lib.basic.val_rdy.ifcs import ValRdySendIfcRTL as SendIfcRTL from ..lib.cmd_type import * +from ..lib.opt_type import OPT_RET from ..lib.util.common import * from ..mem.const.ConstQueueDynamicRTL import ConstQueueDynamicRTL from ..mem.ctrl.CtrlMemDynamicRTL import CtrlMemDynamicRTL @@ -128,6 +129,7 @@ def construct(s, IntraCgraPktType, s.element_done = Wire(1) s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) + s.routing_write_fire = [Wire(b1) for _ in range(num_fu_inports)] s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -224,7 +226,7 @@ def construct(s, IntraCgraPktType, s.register_cluster.write_data_from_routing_crossbar[i] //= \ s.routing_crossbar.send_data[num_tile_outports + i].msg s.register_cluster.write_valid_from_routing_crossbar[i] //= \ - s.routing_crossbar.send_data[num_tile_outports + i].val + s.routing_write_fire[i] s.fu_crossbar.send_data[num_tile_outports + i] //= \ s.register_cluster.recv_data_from_fu_crossbar[i] @@ -235,6 +237,17 @@ def construct(s, IntraCgraPktType, s.element.recv_in[i] s.register_cluster.inport_opt //= s.ctrl_mem.send_ctrl.msg + + @update + def update_routing_write_fire(): + for i in range(num_fu_inports): + s.routing_write_fire[i] @= \ + s.ctrl_mem.send_ctrl.val & \ + (s.ctrl_mem.send_ctrl.msg.operation == OPT_RET) & \ + (s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + s.routing_crossbar.send_data[num_tile_outports + i].val & \ + s.routing_crossbar.send_data[num_tile_outports + i].rdy + # Clear ports are only useful during context switching. # We connect to 0 to make sure they have drivers. for i in range(len(FuList)): From 75751858186203a3f1ef043918a894a9ce30e5ea Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 23 Jul 2026 05:54:27 +0800 Subject: [PATCH 54/60] Consume prologue inputs without routing --- noc/CrossbarRTL.py | 22 +++++----------------- noc/test/CrossbarRTL_test.py | 19 ++++--------------- tile/TileRTL.py | 30 +----------------------------- tile/TileWithContextSwitchRTL.py | 2 -- tile/TileWithStreamingLoadRTL.py | 2 -- 5 files changed, 10 insertions(+), 65 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index e26330db..a9b5863c 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -45,10 +45,6 @@ def construct(s, s.recv_data_val[i] //= s.recv_data[i].val s.crossbar_outport = [InPort(InType) for _ in range(num_outports)] - s.preserve_outport = [InPort(b1) for _ in range(num_outports)] - # Local outputs normally stop participating once the FU is done. - # preserve_outport keeps selected local outputs live when the same control - # step still needs routing-xbar data for a register write or FU operand. s.send_data = [SendIfcRTL(DataType) for _ in range(num_outports)] s.in_dir = [Wire(InType) for _ in range(num_outports)] @@ -62,7 +58,6 @@ def construct(s, s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) s.crossbar_id = InPort(b1) s.compute_done = InPort(b1) - s.drain_when_inactive = InPort(b1) s.ctrl_addr_inport = InPort(CtrlAddrType) @@ -134,7 +129,7 @@ def update_signal(): s.all_send_accepted @= 0 for i in range(num_inports): - s.recv_data[i].rdy @= reduce_and(s.recv_valid_vector) & \ + s.recv_data[i].rdy @= reduce_and(s.recv_valid_or_during_prologue_allowing_vector) & \ s.all_send_accepted & \ s.recv_required_vector[i] @@ -154,12 +149,6 @@ def update_signal(): s.recv_opt.rdy @= s.all_send_accepted & \ reduce_and(s.recv_valid_or_during_prologue_allowing_vector) - elif s.drain_when_inactive: - # Test-only/manual drain path: when no routing op is active, consume - # queued input tokens without producing outputs. Tile-level users tie - # this low, so normal scheduled execution is unchanged. - for i in range(num_inports): - s.recv_data[i].rdy @= 1 @update_ff def update_prologue_counter(): @@ -271,10 +260,9 @@ def update_recv_required_vector(): for i in range(num_outports): if s.in_dir[i] > 0: - # A prologue forwards a warm-up copy without dequeuing this token. - # The following real step consumes it, even after the local FU has - # completed, so it cannot block the upstream tile. - s.recv_required_vector[s.in_dir_local[i]] @= ~s.during_prologue_allowing_vector[i] + # A prologue acknowledges an available input but does not route it. + # This keeps the input stream aligned with the control schedule. + s.recv_required_vector[s.in_dir_local[i]] @= 1 @update def update_send_required_vector(): @@ -283,7 +271,7 @@ def update_send_required_vector(): s.send_required_vector[i] @= 0 for i in range(num_outports): - if s.in_dir[i] > 0: + if (s.in_dir[i] > 0) & ~s.during_prologue_allowing_vector[i]: s.send_required_vector[i] @= 1 diff --git a/noc/test/CrossbarRTL_test.py b/noc/test/CrossbarRTL_test.py index 314e8ae4..5f7a63d4 100644 --- a/noc/test/CrossbarRTL_test.py +++ b/noc/test/CrossbarRTL_test.py @@ -25,7 +25,6 @@ class TestHarness(Component): def construct(s, CrossbarUnit, DataType, CtrlType, num_inports, num_outports, src_data, src_routing, sink_out, prologue_counts = None, - drain_when_inactive = 0, ctrl_addr_sequence = None): num_tiles = 1 @@ -55,7 +54,6 @@ def construct(s, CrossbarUnit, DataType, CtrlType, s.dut.prologue_count_inport[addr][i] //= count s.src_opt.send //= s.dut.recv_opt s.dut.compute_done //= 0 - s.dut.drain_when_inactive //= drain_when_inactive s.ctrl_addr_sequence = ctrl_addr_sequence or [0] @update @@ -160,26 +158,17 @@ def test_multi_cast(): num_routing_outports, src_data, src_opt, sink_out) run_sim(th) -def test_prologue_warms_and_preserves_input(): +def test_prologue_consumes_without_routing(): src_opt = [CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)]), CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)])] - src_data = [[DataType(7, 1)], [], []] - # The prologue emits a warm-up copy; the following real step consumes it. - sink_out = [[DataType(7, 1), DataType(7, 1)], [], []] + src_data = [[DataType(7, 1), DataType(7, 1)], [], []] + # The prologue consumes the first token without sending it downstream. + sink_out = [[DataType(7, 1)], [], []] th = TestHarness(FU, DataType, CtrlType, num_tile_inports, num_routing_outports, src_data, src_opt, sink_out, prologue_counts = {(0, 0): 1}) run_sim(th) - -def test_inactive_drain_consumes_input(): - src_opt = [] - src_data = [[DataType(11, 1)], [], []] - sink_out = [[], [], []] - th = TestHarness(FU, DataType, CtrlType, num_tile_inports, - num_routing_outports, src_data, src_opt, sink_out, - drain_when_inactive = 1) - run_sim(th) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 3d3f29eb..0facddcf 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -31,7 +31,6 @@ from ..noc.CrossbarRTL import CrossbarRTL from ..noc.LinkOrRTL import LinkOrRTL from ..noc.PyOCN.pymtl3_net.channel.ChannelRTL import ChannelRTL -from ..lib.basic.val_rdy.queues import PipeQueueRTL from ..rf.RegisterRTL import RegisterRTL from ..lib.util.data_struct_attr import * @@ -117,7 +116,7 @@ def construct(s, IntraCgraPktType, # The `tile_in_channel` indicates the outport channels that are # connected to the next tiles. - s.tile_in_channel = [ChannelRTL(DataType, QueueType=PipeQueueRTL, latency = 1) + s.tile_in_channel = [ChannelRTL(DataType, latency = 1) for _ in range(num_tile_inports)] # The `tile_out_or_link` would "or" the outports of the @@ -129,7 +128,6 @@ def construct(s, IntraCgraPktType, s.element_done = Wire(1) s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) - s.routing_preserve_local = [Wire(b1) for _ in range(num_fu_inports)] s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -202,30 +200,6 @@ def construct(s, IntraCgraPktType, s.fu_crossbar.crossbar_outport[i] //= \ s.ctrl_mem.send_ctrl.msg.fu_xbar_outport[i] - # Only routing-xbar outputs that target the local register/FU side need - # preservation. Tile-to-tile outputs still follow normal compute_done - # gating because downstream tiles consume them through send_data. - if i < num_tile_outports: - s.routing_crossbar.preserve_outport[i] //= 0 - s.fu_crossbar.preserve_outport[i] //= 0 - else: - local_idx = i - num_tile_outports - s.routing_crossbar.preserve_outport[i] //= \ - s.routing_preserve_local[local_idx] - s.fu_crossbar.preserve_outport[i] //= 0 - - - @update - def update_routing_preserve_local(): - for i in range(num_fu_inports): - read_towards = s.ctrl_mem.send_ctrl.msg.read_reg_towards[i] - read_towards_routing = \ - (read_towards == READ_TOWARDS_ROUTING_XBAR) | \ - (read_towards == READ_TOWARDS_BOTH) - s.routing_preserve_local[i] @= \ - (s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR) | \ - read_towards_routing - # Connections on the `fu_crossbar`. for i in range(num_fu_outports): s.element.send_out[i] //= s.fu_crossbar.recv_data[i] @@ -346,8 +320,6 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done - s.routing_crossbar.drain_when_inactive @= 0 - s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): diff --git a/tile/TileWithContextSwitchRTL.py b/tile/TileWithContextSwitchRTL.py index a98c74e5..284fe62f 100644 --- a/tile/TileWithContextSwitchRTL.py +++ b/tile/TileWithContextSwitchRTL.py @@ -364,8 +364,6 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done - s.routing_crossbar.drain_when_inactive @= 0 - s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): diff --git a/tile/TileWithStreamingLoadRTL.py b/tile/TileWithStreamingLoadRTL.py index babbed35..aeb3f97a 100644 --- a/tile/TileWithStreamingLoadRTL.py +++ b/tile/TileWithStreamingLoadRTL.py @@ -313,8 +313,6 @@ def already_done(): def notify_crossbars_compute_status(): s.routing_crossbar.compute_done @= s.element_done s.fu_crossbar.compute_done @= s.element_done - s.routing_crossbar.drain_when_inactive @= 0 - s.fu_crossbar.drain_when_inactive @= 0 # Line trace def line_trace(s): From d667b38d66bea73ff1d74d64715b4bd27421c4c2 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:50:18 +0800 Subject: [PATCH 55/60] Keep prologue inputs queued --- noc/CrossbarRTL.py | 6 +++--- noc/test/CrossbarRTL_test.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index a9b5863c..e6bcc060 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -260,9 +260,9 @@ def update_recv_required_vector(): for i in range(num_outports): if s.in_dir[i] > 0: - # A prologue acknowledges an available input but does not route it. - # This keeps the input stream aligned with the control schedule. - s.recv_required_vector[s.in_dir_local[i]] @= 1 + # A prologue advances control without routing or dequeuing its input. + s.recv_required_vector[s.in_dir_local[i]] @= \ + ~s.during_prologue_allowing_vector[i] @update def update_send_required_vector(): diff --git a/noc/test/CrossbarRTL_test.py b/noc/test/CrossbarRTL_test.py index 5f7a63d4..bbfb8016 100644 --- a/noc/test/CrossbarRTL_test.py +++ b/noc/test/CrossbarRTL_test.py @@ -158,15 +158,15 @@ def test_multi_cast(): num_routing_outports, src_data, src_opt, sink_out) run_sim(th) -def test_prologue_consumes_without_routing(): +def test_prologue_skips_routing(): src_opt = [CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)]), CtrlType(OPT_ADD, pickRegister, [TileInType(1), TileInType(0), TileInType(0)], [FuOutType(0), FuOutType(0), FuOutType(0)])] - src_data = [[DataType(7, 1), DataType(7, 1)], [], []] - # The prologue consumes the first token without sending it downstream. + src_data = [[DataType(7, 1)], [], []] + # The prologue neither routes nor dequeues the token. sink_out = [[DataType(7, 1)], [], []] th = TestHarness(FU, DataType, CtrlType, num_tile_inports, num_routing_outports, src_data, src_opt, sink_out, From b19259a5ba86d79cc65f4a0ddd4016ba366dd56d Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:39:26 +0800 Subject: [PATCH 56/60] Avoid PHI waiting on inactive predecessor --- fu/single/PhiRTL.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/fu/single/PhiRTL.py b/fu/single/PhiRTL.py index 64bd394b..cb12e8e7 100644 --- a/fu/single/PhiRTL.py +++ b/fu/single/PhiRTL.py @@ -94,11 +94,22 @@ def comb_logic(): else: # No predecessor is active. s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload s.send_out[0].msg.predicate @= 0 - s.recv_all_val @= ((s.first[s.ctrl_addr_inport] & s.recv_in[s.in0_idx].val) | \ - (~s.first[s.ctrl_addr_inport] & s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val)) + + # After the first PHI_START execution, the selected predecessor is + # identified by its predicate. Do not stall the selected true token + # behind a later predicated-off token from the other predecessor. + if s.first[s.ctrl_addr_inport]: + s.recv_all_val @= s.recv_in[s.in0_idx].val + else: + s.recv_all_val @= \ + (s.recv_in[s.in0_idx].val & s.recv_in[s.in0_idx].msg.predicate) | \ + (s.recv_in[s.in1_idx].val & s.recv_in[s.in1_idx].msg.predicate) | \ + (s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val) s.send_out[0].val @= s.recv_all_val - s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy - s.recv_in[s.in1_idx].rdy @= ~s.first[s.ctrl_addr_inport] & s.recv_all_val & s.send_out[0].rdy + s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy & \ + (s.first[s.ctrl_addr_inport] | s.recv_in[s.in0_idx].val) + s.recv_in[s.in1_idx].rdy @= ~s.first[s.ctrl_addr_inport] & \ + s.recv_all_val & s.send_out[0].rdy & s.recv_in[s.in1_idx].val s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy elif s.recv_opt.msg.operation == OPT_PHI_CONST: From 0b0018f1881ac7a5235a6c52a726ab0a4ed60a54 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:33:47 +0800 Subject: [PATCH 57/60] Generalize routing crossbar register writes --- tile/TileRTL.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index f72df77b..8ee14a53 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -243,7 +243,6 @@ def update_routing_write_fire(): for i in range(num_fu_inports): s.routing_write_fire[i] @= \ s.ctrl_mem.send_ctrl.val & \ - (s.ctrl_mem.send_ctrl.msg.operation == OPT_RET) & \ (s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ s.routing_crossbar.send_data[num_tile_outports + i].val & \ s.routing_crossbar.send_data[num_tile_outports + i].rdy From 8ec94bbe6694fdcb614a7ff857f708cf3f889ed1 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:42:03 +0800 Subject: [PATCH 58/60] Split routing register writes from FU operands --- tile/TileRTL.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 8ee14a53..e7516b4e 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -213,22 +213,30 @@ def construct(s, IntraCgraPktType, # the FUs (via `fu_crossbar`) with the outports of the # `routing_crossbar` through the corresponding channels. for i in range(num_tile_outports): - s.fu_crossbar.send_data[i] //= s.tile_out_or_link[i].recv_fu - s.routing_crossbar.send_data[i] //= s.tile_out_or_link[i].recv_xbar - s.tile_out_or_link[i].send //= s.send_data[i] + s.tile_out_or_link[i].recv_fu.msg //= s.fu_crossbar.send_data[i].msg + s.tile_out_or_link[i].recv_fu.val //= s.fu_crossbar.send_data[i].val + s.tile_out_or_link[i].recv_xbar.msg //= \ + s.routing_crossbar.send_data[i].msg + s.tile_out_or_link[i].recv_xbar.val //= \ + s.routing_crossbar.send_data[i].val + s.send_data[i].msg //= s.tile_out_or_link[i].send.msg + s.send_data[i].val //= s.tile_out_or_link[i].send.val + s.tile_out_or_link[i].send.rdy //= s.send_data[i].rdy # Crossbars outputs are integrated with the "register_cluster". # Whether the required operands for FU are from the "routing_crossbar" # or from the "register_cluster" depends on the control signals. for i in range(num_fu_inports): - s.routing_crossbar.send_data[num_tile_outports + i] //= \ - s.register_cluster.recv_data_from_routing_crossbar[i] s.register_cluster.write_data_from_routing_crossbar[i] //= \ s.routing_crossbar.send_data[num_tile_outports + i].msg s.register_cluster.write_valid_from_routing_crossbar[i] //= \ s.routing_write_fire[i] - s.fu_crossbar.send_data[num_tile_outports + i] //= \ - s.register_cluster.recv_data_from_fu_crossbar[i] + s.register_cluster.recv_data_from_routing_crossbar[i].msg //= \ + s.routing_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.recv_data_from_fu_crossbar[i].msg //= \ + s.fu_crossbar.send_data[num_tile_outports + i].msg + s.register_cluster.recv_data_from_fu_crossbar[i].val //= \ + s.fu_crossbar.send_data[num_tile_outports + i].val s.register_cluster.recv_data_from_const[i].msg //= DataType() s.register_cluster.recv_data_from_const[i].val //= 0 @@ -239,13 +247,31 @@ def construct(s, IntraCgraPktType, @update - def update_routing_write_fire(): + def update_routing_to_reg_inputs(): + for i in range(num_tile_outports): + s.routing_crossbar.send_data[i].rdy @= s.send_data[i].rdy for i in range(num_fu_inports): + is_reg_write = \ + s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR + s.register_cluster.recv_data_from_routing_crossbar[i].val @= \ + s.routing_crossbar.send_data[num_tile_outports + i].val & \ + ~is_reg_write s.routing_write_fire[i] @= \ s.ctrl_mem.send_ctrl.val & \ - (s.ctrl_mem.send_ctrl.msg.write_reg_from[i] == PORT_ROUTING_CROSSBAR) & \ + is_reg_write & \ s.routing_crossbar.send_data[num_tile_outports + i].val & \ s.routing_crossbar.send_data[num_tile_outports + i].rdy + s.routing_crossbar.send_data[num_tile_outports + i].rdy @= \ + is_reg_write | \ + s.register_cluster.recv_data_from_routing_crossbar[i].rdy + + @update + def update_reg_cluster_input_rdy(): + for i in range(num_tile_outports): + s.fu_crossbar.send_data[i].rdy @= s.send_data[i].rdy + for i in range(num_fu_inports): + s.fu_crossbar.send_data[num_tile_outports + i].rdy @= \ + s.register_cluster.recv_data_from_fu_crossbar[i].rdy # Clear ports are only useful during context switching. # We connect to 0 to make sure they have drivers. From 3da92202183c93249e68bb66fbc90b11fe7c749d Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:54:23 +0800 Subject: [PATCH 59/60] Preserve live routing outputs after FU completion --- noc/CrossbarRTL.py | 17 +++++++++++++---- tile/TileRTL.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/noc/CrossbarRTL.py b/noc/CrossbarRTL.py index e6bcc060..45e0c1bf 100644 --- a/noc/CrossbarRTL.py +++ b/noc/CrossbarRTL.py @@ -46,6 +46,7 @@ def construct(s, s.crossbar_outport = [InPort(InType) for _ in range(num_outports)] s.send_data = [SendIfcRTL(DataType) for _ in range(num_outports)] + s.preserve_outport = [InPort(b1) for _ in range(num_outports)] s.in_dir = [Wire(InType) for _ in range(num_outports)] s.in_dir_local = [Wire(NumInportType) for _ in range(num_outports)] @@ -239,7 +240,8 @@ def update_rdy_vector(): # After the FU finishes, only directions toward a local consumer still # backpressure the crossbar; routed outputs may drain independently. if (s.in_dir[i] > 0) & \ - (~s.compute_done | (i < outport_towards_local_base_id)): + (~s.compute_done | (i < outport_towards_local_base_id) | \ + s.preserve_outport[i]): s.send_rdy_vector[i] @= s.send_data[i].rdy else: s.send_rdy_vector[i] @= 1 @@ -248,7 +250,9 @@ def update_rdy_vector(): def update_valid_vector(): s.recv_valid_vector @= 0 for i in range(num_outports): - if s.in_dir[i] > 0: + if (s.in_dir[i] > 0) & \ + (~s.compute_done | (i < outport_towards_local_base_id) | \ + s.preserve_outport[i]): s.recv_valid_vector[i] @= s.recv_data_val[s.in_dir_local[i]] else: s.recv_valid_vector[i] @= 1 @@ -259,7 +263,9 @@ def update_recv_required_vector(): s.recv_required_vector[i] @= 0 for i in range(num_outports): - if s.in_dir[i] > 0: + if (s.in_dir[i] > 0) & \ + (~s.compute_done | (i < outport_towards_local_base_id) | \ + s.preserve_outport[i]): # A prologue advances control without routing or dequeuing its input. s.recv_required_vector[s.in_dir_local[i]] @= \ ~s.during_prologue_allowing_vector[i] @@ -271,7 +277,10 @@ def update_send_required_vector(): s.send_required_vector[i] @= 0 for i in range(num_outports): - if (s.in_dir[i] > 0) & ~s.during_prologue_allowing_vector[i]: + if (s.in_dir[i] > 0) & \ + ~s.during_prologue_allowing_vector[i] & \ + (~s.compute_done | (i < outport_towards_local_base_id) | \ + s.preserve_outport[i]): s.send_required_vector[i] @= 1 diff --git a/tile/TileRTL.py b/tile/TileRTL.py index e7516b4e..5641c34a 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -201,6 +201,18 @@ def construct(s, IntraCgraPktType, s.ctrl_mem.send_ctrl.msg.routing_xbar_outport[i] s.fu_crossbar.crossbar_outport[i] //= \ s.ctrl_mem.send_ctrl.msg.fu_xbar_outport[i] + if i < num_tile_outports: + s.routing_crossbar.preserve_outport[i] //= 0 + s.fu_crossbar.preserve_outport[i] //= 0 + else: + local_idx = i - num_tile_outports + read_towards = s.ctrl_mem.send_ctrl.msg.read_reg_towards[local_idx] + s.routing_crossbar.preserve_outport[i] //= \ + (s.ctrl_mem.send_ctrl.msg.write_reg_from[local_idx] == \ + PORT_ROUTING_CROSSBAR) | \ + (read_towards == READ_TOWARDS_ROUTING_XBAR) | \ + (read_towards == READ_TOWARDS_BOTH) + s.fu_crossbar.preserve_outport[i] //= 0 # Connections on the `fu_crossbar`. for i in range(num_fu_outports): From 9fd96cdf6bde771b4578cf377221e6c6709b64a6 Mon Sep 17 00:00:00 2001 From: guosran <165251838+guosran@users.noreply.github.com> Date: Sat, 25 Jul 2026 02:46:51 +0800 Subject: [PATCH 60/60] Avoid final routing register overwrite --- tile/TileRTL.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/tile/TileRTL.py b/tile/TileRTL.py index 5641c34a..e622f639 100644 --- a/tile/TileRTL.py +++ b/tile/TileRTL.py @@ -63,6 +63,7 @@ def construct(s, IntraCgraPktType, CtrlAddrType = mk_bits(clog2(ctrl_mem_size)) DataAddrType = mk_bits(clog2(data_mem_size)) + RegBankIdxType = mk_bits(clog2(num_fu_inports)) # Interfaces. s.recv_data = [RecvIfcRTL(DataType) @@ -130,6 +131,8 @@ def construct(s, IntraCgraPktType, s.fu_crossbar_done = Wire(1) s.routing_crossbar_done = Wire(1) s.routing_write_fire = [Wire(b1) for _ in range(num_fu_inports)] + s.routing_preserve_outport = [Wire(b1) + for _ in range(num_routing_xbar_outports)] s.cgra_id = InPort(mk_bits(max(1, clog2(num_cgras)))) s.tile_id = InPort(mk_bits(clog2(num_tiles + 1))) @@ -201,18 +204,24 @@ def construct(s, IntraCgraPktType, s.ctrl_mem.send_ctrl.msg.routing_xbar_outport[i] s.fu_crossbar.crossbar_outport[i] //= \ s.ctrl_mem.send_ctrl.msg.fu_xbar_outport[i] - if i < num_tile_outports: - s.routing_crossbar.preserve_outport[i] //= 0 - s.fu_crossbar.preserve_outport[i] //= 0 - else: - local_idx = i - num_tile_outports - read_towards = s.ctrl_mem.send_ctrl.msg.read_reg_towards[local_idx] - s.routing_crossbar.preserve_outport[i] //= \ - (s.ctrl_mem.send_ctrl.msg.write_reg_from[local_idx] == \ - PORT_ROUTING_CROSSBAR) | \ - (read_towards == READ_TOWARDS_ROUTING_XBAR) | \ - (read_towards == READ_TOWARDS_BOTH) - s.fu_crossbar.preserve_outport[i] //= 0 + s.routing_crossbar.preserve_outport[i] //= \ + s.routing_preserve_outport[i] + s.fu_crossbar.preserve_outport[i] //= 0 + + @update + def up_routing_preserve_outport(): + for i in range(num_routing_xbar_outports): + if i < num_tile_outports: + s.routing_preserve_outport[i] @= 0 + else: + local_idx = RegBankIdxType(i - num_tile_outports) + read_towards = s.ctrl_mem.send_ctrl.msg.read_reg_towards[local_idx] + s.routing_preserve_outport[i] @= \ + (s.ctrl_mem.send_ctrl.msg.operation == OPT_RET) & \ + ((s.ctrl_mem.send_ctrl.msg.write_reg_from[local_idx] == \ + PORT_ROUTING_CROSSBAR) | \ + (read_towards == READ_TOWARDS_ROUTING_XBAR) | \ + (read_towards == READ_TOWARDS_BOTH)) # Connections on the `fu_crossbar`. for i in range(num_fu_outports): @@ -270,6 +279,8 @@ def update_routing_to_reg_inputs(): ~is_reg_write s.routing_write_fire[i] @= \ s.ctrl_mem.send_ctrl.val & \ + (~s.ctrl_mem.send_ctrl.msg.is_last_ctrl | \ + (s.ctrl_mem.send_ctrl.msg.operation == OPT_RET)) & \ is_reg_write & \ s.routing_crossbar.send_data[num_tile_outports + i].val & \ s.routing_crossbar.send_data[num_tile_outports + i].rdy