Fix/conv all fixes

cgra/test/CgraRTL_fir_2x2_loop_counter_test.py

@@ -351,7 +351,7 @@ def sim_fir_with_loop_counter(cmdline_opts, mem_access_is_combinational, has_ctr
   # the stored result should still be correct thanks to the grant predicate.
   kTotalCtrlSteps = kCtrlCountPerIter * \
                     (kLoopUpperBound - kLoopLowerBound) + \
-                    10
+                    13
   kExpectedOutput = 2215
 
   # Corresponding DFG:

cgra/test/CgraVerifAssert_test.py

@@ -302,7 +302,7 @@ def line_trace(s):
 
 
 // response should be data_mem[0:4] = [0,1,1,0,1]
-// return value should be i (i = 4 in the end)
+// return value should be the final loop index (i = 5 at exit)
 '''
 
 def sim_fir_return(cmdline_opts, mem_access_is_combinational, has_ctrl_ring):
@@ -320,12 +320,13 @@ def sim_fir_return(cmdline_opts, mem_access_is_combinational, has_ctrl_ring):
   kCtrlCountPerIter = 5
   kCmpOne = 1
   kCmpZero = 0
-  # Though kTotalCtrlSteps is way more than required loop iteration count,
-  # the stored result should still be correct thanks to the grant predicate.
+  # Leave a small drain window after the loop so the final RET and memory
+  # assertions can complete without delaying return by hundreds of cycles.
+  # With final-RET semantics, the returned value is the loop-exit index.
   kTotalCtrlSteps = kCtrlCountPerIter * \
                     (kLoopUpperBound - kLoopLowerBound) + \
-                    1000
-  kExpectedOutput = 4
+                    3
+  kExpectedOutput = 5
 
   # More details are shown in:
   # https://github.com/tancheng/VectorCGRA/tree/master/doc/figures/assert_test/DFG.png.

cgra/test/CgraWithContextSwitchRTL_test.py

@@ -346,14 +346,15 @@ def sim_fir_return_two_tasks(cmdline_opts, mem_access_is_combinational):
   kLoopUpperBound_Task2 = 9
   kCtrlCountPerIter_Task1 = 4
   kCtrlCountPerIter_Task2 = 3
-  # Though kTotalCtrlSteps is way more than required loop iteration count,
-  # the stored result should still be correct thanks to the grant predicate.
+  # Leave a bounded drain window after each task so final RET observes the
+  # completed result without delaying context-switch return by hundreds of
+  # cycles.
   kTotalCtrlSteps_Task1 = kCtrlCountPerIter_Task1 * \
                           (kLoopUpperBound_Task1 - kLoopLowerBound) + \
-                          100
+                          13
   kTotalCtrlSteps_Task2 = kCtrlCountPerIter_Task2 * \
                           (kLoopUpperBound_Task2 - kLoopLowerBound) + \
-                          100
+                          13
   kExpectedOutput_Task1 = 2215
   kExpectedOutput_Task2 = 1816
   src_opt_pkt = [

controller/ControllerRTL.py

@@ -201,6 +201,7 @@ def construct(s,
     s.dma_spm_addr     = Wire(DmaSpmAddrType)
     s.dma_bytes        = Wire(DmaBytesType)
     s.dma_tag          = Wire(DmaTagType)
+    s.has_ret_kernel   = Wire(b1)
 
     # Connections.
     # Requests towards others, 1 cycle delay to improve timing.
@@ -231,6 +232,20 @@ def construct(s,
       s.recv_from_im2col_pkt_queue.recv.val //= 0
       s.recv_from_im2col_pkt_queue.recv.msg //= IntraCgraPktType()
 
+    @update_ff
+    def update_has_ret_kernel():
+      if s.reset:
+        s.has_ret_kernel <<= 0
+      else:
+        cpu_payload = s.recv_from_cpu_pkt_queue.send.msg.payload
+        is_ret_config = (cpu_payload.cmd == CMD_CONFIG) & (
+            (cpu_payload.ctrl.operation == OPT_RET) |
+            (cpu_payload.ctrl.operation == OPT_RET_VOID))
+        config_accepted = (s.recv_from_cpu_pkt_queue.send.val &
+                           s.recv_from_cpu_pkt_queue.send.rdy)
+        if config_accepted & is_ret_config:
+          s.has_ret_kernel <<= 1
+
     @update_ff
     def update_dma_cmd_regs():
       if s.reset:
@@ -451,6 +466,7 @@ def update_received_msg():
 
       # For the load request from NoC.
       received_pkt = s.recv_from_inter_cgra_noc.msg
+      is_empty_ret_complete = b1(0)
       if s.recv_from_inter_cgra_noc.val:
         if s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_LOAD_REQUEST:
           s.send_to_mem_load_request_queue.recv.val @= 1
@@ -493,20 +509,25 @@ def update_received_msg():
             s.send_to_tile_load_response_queue.recv.val @= 1
 
         elif s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_COMPLETE:
-          s.recv_from_inter_cgra_noc.rdy @= s.send_to_cpu_pkt_queue.recv.rdy
-          s.send_to_cpu_pkt_queue.recv.val @= 1
-          s.send_to_cpu_pkt_queue.recv.msg @= \
-              IntraCgraPktType(s.recv_from_inter_cgra_noc.msg.src_tile_id, # src
-                               s.recv_from_inter_cgra_noc.msg.dst_tile_id, # dst
-                               s.recv_from_inter_cgra_noc.msg.src, # src_cgra_id
-                               s.recv_from_inter_cgra_noc.msg.dst, # src_cgra_id
-                               s.recv_from_inter_cgra_noc.msg.src_x, # src_cgra_x
-                               s.recv_from_inter_cgra_noc.msg.src_y, # src_cgra_y
-                               s.recv_from_inter_cgra_noc.msg.dst_x, # dst_cgra_x
-                               s.recv_from_inter_cgra_noc.msg.dst_y, # dst_cgra_y
-                               0, # opaque
-                               0, # vc_id
-                               s.recv_from_inter_cgra_noc.msg.payload)
+          is_empty_ret_complete = (s.has_ret_kernel &
+              ~s.recv_from_inter_cgra_noc.msg.payload.data.predicate)
+          if is_empty_ret_complete:
+            s.recv_from_inter_cgra_noc.rdy @= 1
+          else:
+            s.recv_from_inter_cgra_noc.rdy @= s.send_to_cpu_pkt_queue.recv.rdy
+            s.send_to_cpu_pkt_queue.recv.val @= 1
+            s.send_to_cpu_pkt_queue.recv.msg @= IntraCgraPktType(
+                s.recv_from_inter_cgra_noc.msg.src_tile_id, # src
+                s.recv_from_inter_cgra_noc.msg.dst_tile_id, # dst
+                s.recv_from_inter_cgra_noc.msg.src, # src_cgra_id
+                s.recv_from_inter_cgra_noc.msg.dst, # src_cgra_id
+                s.recv_from_inter_cgra_noc.msg.src_x, # src_cgra_x
+                s.recv_from_inter_cgra_noc.msg.src_y, # src_cgra_y
+                s.recv_from_inter_cgra_noc.msg.dst_x, # dst_cgra_x
+                s.recv_from_inter_cgra_noc.msg.dst_y, # dst_cgra_y
+                0, # opaque
+                0, # vc_id
+                s.recv_from_inter_cgra_noc.msg.payload)
 
         # Consume and discard the leaf counter complete signal (loop termination
         # notification from LoopCounter FU) to avoid blocking the NoC.
@@ -544,7 +565,11 @@ def update_received_msg():
              (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_LAUNCH) | \
              (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_LOWER) | \
              (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_UPPER) | \
-             (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) :
+             (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_LOOP_STEP) | \
+             (s.recv_from_inter_cgra_noc.msg.payload.cmd == CMD_CONFIG_GEP_STRIDE) :
+          # These packets are tile-local configuration, including GEP stride.
+          # Forward them on the control ring; treating them as controller-local
+          # commands would leave the target tile running GEP with a stale stride.
           s.recv_from_inter_cgra_noc.rdy @= s.send_to_ctrl_ring_pkt.rdy
           s.send_to_ctrl_ring_pkt.val @= s.recv_from_inter_cgra_noc.val
           s.send_to_ctrl_ring_pkt.msg @= \

fu/single/AdderRTL.py

@@ -67,9 +67,23 @@ def comb_logic():
       if s.recv_opt.val:
         if s.recv_opt.msg.operation == OPT_ADD:
           s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload + s.recv_in[s.in1_idx].msg.payload
-          s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \
-                                         s.recv_in[s.in1_idx].msg.predicate & \
-                                         s.reached_vector_factor
+          # Treat a predicated-off zero as the additive identity: a zero value with
+          # predicate=0 is a neutral placeholder, not a missing contributing
+          # operand. Example: (value=7,pred=1) + (value=0,pred=0) should
+          # produce pred=1, while (value=7,pred=1) + (value=5,pred=0) stays
+          # pred=0.
+          if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate:
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          elif s.recv_in[s.in0_idx].msg.predicate & \
+               ~s.recv_in[s.in1_idx].msg.predicate & \
+               (s.recv_in[s.in1_idx].msg.payload == s.const_zero.payload):
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          elif s.recv_in[s.in1_idx].msg.predicate & \
+               ~s.recv_in[s.in0_idx].msg.predicate & \
+               (s.recv_in[s.in0_idx].msg.payload == s.const_zero.payload):
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          else:
+            s.send_out[0].msg.predicate @= 0
           s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val
           s.send_out[0].val @= s.recv_all_val
           s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy
@@ -99,9 +113,20 @@ def comb_logic():
 
         elif s.recv_opt.msg.operation == OPT_SUB:
           s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload - s.recv_in[s.in1_idx].msg.payload
-          s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \
-                                         s.recv_in[s.in1_idx].msg.predicate & \
-                                         s.reached_vector_factor
+          # Same identity rule for SUB: subtracting an inactive zero keeps
+          # the active operand live; subtracting an inactive nonzero does not.
+          if s.recv_in[s.in0_idx].msg.predicate & s.recv_in[s.in1_idx].msg.predicate:
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          elif s.recv_in[s.in0_idx].msg.predicate & \
+               ~s.recv_in[s.in1_idx].msg.predicate & \
+               (s.recv_in[s.in1_idx].msg.payload == s.const_zero.payload):
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          elif s.recv_in[s.in1_idx].msg.predicate & \
+               ~s.recv_in[s.in0_idx].msg.predicate & \
+               (s.recv_in[s.in0_idx].msg.payload == s.const_zero.payload):
+            s.send_out[0].msg.predicate @= s.reached_vector_factor
+          else:
+            s.send_out[0].msg.predicate @= 0
           s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val
           s.send_out[0].val @= s.recv_all_val
           s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy

fu/single/DivRTL.py

@@ -32,7 +32,38 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0
     s.in0_idx //= s.in0[0:idx_nbits]
     s.in1_idx //= s.in1[0:idx_nbits]
 
+    PayloadType = s.DataType.get_field_type('payload')
+    RemainderType = mk_bits(PayloadType.nbits + 1)
+
     s.recv_all_val = Wire(1)
+    s.dividend = Wire(PayloadType)
+    s.divisor = Wire(PayloadType)
+    s.div_quotient = Wire(PayloadType)
+    s.div_remainder = Wire(PayloadType)
+
+    # Compute quotient and remainder with shift/subtract logic instead of
+    # Python percent/modulo so Verilator translation sees only fixed-width RTL
+    # operations. Divisor zero is defined as quotient=0 and remainder=0,
+    # matching the deterministic zero-divisor behavior used by the tests.
+    @update
+    def comb_div_rem():
+      quotient = PayloadType(0)
+      remainder = RemainderType(0)
+
+      if s.divisor != 0:
+        for i in range(PayloadType.nbits):
+          remainder = (remainder << 1) | \
+              zext(s.dividend[PayloadType.nbits - 1 - i], RemainderType.nbits)
+          if remainder >= zext(s.divisor, RemainderType.nbits):
+            remainder = remainder - zext(s.divisor, RemainderType.nbits)
+            quotient = quotient | \
+                (PayloadType(1) << (PayloadType.nbits - 1 - i))
+
+      s.div_quotient @= quotient
+      if s.divisor != 0:
+        s.div_remainder @= trunc(remainder, PayloadType)
+      else:
+        s.div_remainder @= 0
 
     @update
     def comb_logic():
@@ -49,6 +80,8 @@ def comb_logic():
 
       s.recv_const.rdy @= 0
       s.recv_opt.rdy @= 0
+      s.dividend @= 0
+      s.divisor @= 0
 
       s.send_to_ctrl_mem.val @= 0
       s.send_to_ctrl_mem.msg @= s.CgraPayloadType(0, 0, 0, 0, 0)
@@ -62,7 +95,9 @@ def comb_logic():
 
       if s.recv_opt.val:
         if s.recv_opt.msg.operation == OPT_DIV:
-          s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload // s.recv_in[s.in1_idx].msg.payload
+          s.dividend @= s.recv_in[s.in0_idx].msg.payload
+          s.divisor @= s.recv_in[s.in1_idx].msg.payload
+          s.send_out[0].msg.payload @= s.div_quotient
           s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \
                                          s.recv_in[s.in1_idx].msg.predicate & \
                                          s.reached_vector_factor
@@ -73,7 +108,9 @@ def comb_logic():
           s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
 
         elif s.recv_opt.msg.operation == OPT_DIV_CONST:
-          s.send_out[0].msg.payload @= s.recv_in[s.in0_idx].msg.payload // s.recv_const.msg.payload
+          s.dividend @= s.recv_in[s.in0_idx].msg.payload
+          s.divisor @= s.recv_const.msg.payload
+          s.send_out[0].msg.payload @= s.div_quotient
           s.send_out[0].msg.predicate @= s.recv_in[s.in0_idx].msg.predicate & \
                                          s.reached_vector_factor
           s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_const.val
@@ -82,6 +119,36 @@ def comb_logic():
           s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy
           s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
 
+        elif s.recv_opt.msg.operation == OPT_REM:
+          s.dividend @= s.recv_in[s.in0_idx].msg.payload
+          s.divisor @= s.recv_in[s.in1_idx].msg.payload
+          s.send_out[0].msg.payload @= s.div_remainder
+          s.send_out[0].msg.predicate @= \
+              s.recv_in[s.in0_idx].msg.predicate & \
+              s.recv_in[s.in1_idx].msg.predicate & \
+              s.reached_vector_factor
+          s.recv_all_val @= \
+              s.recv_in[s.in0_idx].val & s.recv_in[s.in1_idx].val
+          s.send_out[0].val @= s.recv_all_val
+          s.recv_in[s.in0_idx].rdy @= \
+              s.recv_all_val & s.send_out[0].rdy
+          s.recv_in[s.in1_idx].rdy @= \
+              s.recv_all_val & s.send_out[0].rdy
+          s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
+
+        elif s.recv_opt.msg.operation == OPT_REM_CONST:
+          s.dividend @= s.recv_in[s.in0_idx].msg.payload
+          s.divisor @= s.recv_const.msg.payload
+          s.send_out[0].msg.payload @= s.div_remainder
+          s.send_out[0].msg.predicate @= \
+              s.recv_in[s.in0_idx].msg.predicate & s.reached_vector_factor
+          s.recv_all_val @= s.recv_in[s.in0_idx].val & s.recv_const.val
+          s.send_out[0].val @= s.recv_all_val
+          s.recv_in[s.in0_idx].rdy @= \
+              s.recv_all_val & s.send_out[0].rdy
+          s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy
+          s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
+
         else:
           for j in range(num_outports):
             s.send_out[j].val @= b1(0)

fu/single/GepRTL.py

@@ -159,7 +159,10 @@ def comb_logic():
     @update_ff
     def update_stride():
       if s.reset:
-        s.stride <<= s.DataType(0, 0)
+        # DataType has payload/predicate/bypass/delay fields. Use the default
+        # constructor so Verilator sees all fields reset, instead of passing
+        # only payload/predicate.
+        s.stride <<= s.DataType()
       else:
         if s.recv_from_ctrl_mem.val & \
            (s.recv_from_ctrl_mem.msg.cmd == CMD_CONFIG_GEP_STRIDE):

fu/single/GrantRTL.py

@@ -30,7 +30,12 @@ def construct(s, CtrlPktType, num_inports, num_outports, vector_factor_power = 0
     s.in0_idx = Wire(idx_nbits)
     s.in1_idx = Wire(idx_nbits)
     s.recv_all_val = Wire(1)
-    s.already_grt_once = Wire(1)
+    # Per-slot (per ctrl_addr) latch so that multiple GRANT_ONCE ops in the
+    # same tile do not share state. Example: ctrl slot 2 can grant loop bound
+    # A once, while ctrl slot 5 independently grants loop bound B once.
+    num_slots = 1 << s.CtrlAddrType.nbits
+    s.already_grt_once = [Wire(1) for _ in range(num_slots)]
+    s.cur_already_grt_once = Wire(1)
 
     # Connections.
     s.in0_idx //= s.in0[0:idx_nbits]
@@ -56,6 +61,9 @@ def comb_logic():
       s.send_to_ctrl_mem.msg @= s.CgraPayloadType(0, 0, 0, 0, 0)
       s.recv_from_ctrl_mem.rdy @= 0
 
+      # Select the per-slot "already granted" bit for the current ctrl_addr.
+      s.cur_already_grt_once @= s.already_grt_once[s.ctrl_addr_inport]
+
       if s.recv_opt.val:
         if s.recv_opt.msg.fu_in[0] != FuInType(0):
           s.in0 @= s.recv_opt.msg.fu_in[0] - FuInType(1)
@@ -100,13 +108,27 @@ def comb_logic():
           # GRANT_ONCE is used to apply `true` predicate onto a value only once. This
           # is usually used for the constant declared in the entry block of a function.
           s.send_out[0].msg @= s.recv_in[s.in0_idx].msg
-          # Only updates predicate as true for the first time.
-          s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.already_grt_once
+          # Only updates predicate as true for the first time at this ctrl_addr.
+          s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.cur_already_grt_once
 
           s.recv_all_val @= s.recv_in[s.in0_idx].val
           s.send_out[0].val @= s.recv_all_val
           s.recv_in[s.in0_idx].rdy @= s.recv_all_val & s.send_out[0].rdy
           s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
+        elif s.recv_opt.msg.operation == OPT_GRT_ONCE_CONST:
+          # GRANT_ONCE_CONST: every real execution consumes the next const so
+          # the queue remains aligned with the control stream, but only the
+          # first execution for this ctrl_addr emits predicate=1. Example: two
+          # different loop-header constants in the same tile use different
+          # ctrl_addr slots; granting one must not suppress the other. Repeated
+          # executions of the same slot emit predicates 1, 0, 0, ...
+          s.send_out[0].msg @= s.recv_const.msg
+          s.send_out[0].msg.predicate @= s.reached_vector_factor & ~s.cur_already_grt_once
+
+          s.recv_all_val @= s.recv_const.val
+          s.send_out[0].val @= s.recv_all_val
+          s.recv_const.rdy @= s.recv_all_val & s.send_out[0].rdy
+          s.recv_opt.rdy @= s.recv_all_val & s.send_out[0].rdy
 
         else:
           for j in range( num_outports ):
@@ -117,10 +139,19 @@ def comb_logic():
 
     @update_ff
     def record_grt_once():
-      if s.reset | s.clear:
-        s.already_grt_once <<= 0
-      else:
-        if ~s.already_grt_once & s.send_out[0].val & s.send_out[0].rdy & (s.recv_opt.msg.operation == OPT_GRT_ONCE):
-          s.already_grt_once <<= 1
+      for k in range(num_slots):
+        if s.reset | s.clear:
+          s.already_grt_once[k] <<= 0
         else:
-          s.already_grt_once <<= s.already_grt_once
+          # Latch only the slot that just fired a successful GRANT_ONCE /
+          # GRANT_ONCE_CONST; all other slots hold their state. This gives
+          # each ctrl_addr an independent "already granted" flag so multiple
+          # GRANT_ONCE ops in the same tile do not share state.
+          if (s.ctrl_addr_inport == s.CtrlAddrType(k)) & \
+             ~s.already_grt_once[k] & \
+             s.send_out[0].val & s.send_out[0].rdy & \
+             ((s.recv_opt.msg.operation == OPT_GRT_ONCE) | \
+              (s.recv_opt.msg.operation == OPT_GRT_ONCE_CONST)):
+            s.already_grt_once[k] <<= 1
+          else:
+            s.already_grt_once[k] <<= s.already_grt_once[k]