ucie clk distribution

src/GenUcieHeader.scala

@@ -29,19 +29,21 @@ object GenUcieHeader {
   def render(): String = {
     val ports = uciePorts
     // The header holds one register layout, so every port has to agree on
-    // everything that layout depends on. Address and moduleId do not reach it:
-    // the offsets are relative to each port's own MMIO base, and the moduleId
-    // only names modules.
+    // everything that layout depends on. Address, moduleId and the clock
+    // distribution netlist do not reach it: the offsets are relative to each
+    // port's own MMIO base, the moduleId only names modules, and the netlist is
+    // physical design.
     val canonical = ports.map(
       _.copy(
         address = ports.head.address,
-        moduleId = ports.head.moduleId
+        moduleId = ports.head.moduleId,
+        clkDistLayout = ports.head.clkDistLayout
       )
     )
     require(
       canonical.forall(_ == canonical.head),
       "UCIe ports must share a register layout, but their parameters differ " +
-        s"beyond address and moduleId: ${canonical.distinct.mkString("\n")}"
+        s"beyond address, moduleId and clock distribution: ${canonical.distinct.mkString("\n")}"
     )
     edu.berkeley.cs.uciedigital.tilelink.GenUcieHeader
       .render(ports.head, regenCommand)

src/Iris.scala

@@ -44,6 +44,7 @@ class JTAGChipIO(hasReset: Boolean) extends Bundle {
 class IrisSystem(implicit p: Parameters)
     extends edu.berkeley.cs.chippy.ChippySystem
     with testchipip.soc.CanHaveChipletRouting
+    with CanHaveUcieClkRegs // UCIe clock registers, which the router can't attach
     with testchipip.soc.CanHaveSubsystemInjectors // Enables the subsystem injector API
     with testchipip.soc.CanHaveSwitchableOffchipBus // Enables optional off-chip-bus with interface-switch
     with testchipip.serdes.CanHavePeripheryTLSerial
@@ -478,21 +479,32 @@ class IrisConfig(sim: Boolean = false)
           routerParams = testchipip.soc.OffchipRouterParams(tableEntries = 4),
           ports = Seq(
             // PD hardens the two UCIe links together, so UcieComplexPort puts
-            // both instances inside one UcieComplex module. They are identical
-            // logic and share a `moduleId`, so they dedup into a single UcieTL
-            // underneath it; splitting them again is a matter of giving one its
-            // own `moduleId`.
+            // both instances inside one UcieComplex module. Their PHYs differ
+            // in their lane clock distribution: the analog IP's clock DEF gives
+            // each link a buffered clock tree laid out for its own bump field --
+            // link 0's in the canonical orientation, link 1's turned 90 degrees
+            // -- and the netlist of that tree is what its PHY instantiates
+            // outside simulation. So each link is its own module, `moduleId` 0
+            // and 1. Each link also has its own high-speed and digital bypass
+            // clock bumps.
             UcieComplexPort(edu.berkeley.cs.uciedigital.tilelink.UcieTLParams(
               address = 0x200000,
               managerWhere = SBUS,
               numLanes = 16,
-              includeDefaultModels = true
+              includeDefaultModels = true,
+              clkDistLayout =
+                if (sim) edu.berkeley.cs.uciedigital.phy.macros.clocking.ClkDistLayout.Behavioral
+                else edu.berkeley.cs.uciedigital.phy.macros.clocking.ClkDistLayout.Buffered("r0")
             )),
             UcieComplexPort(edu.berkeley.cs.uciedigital.tilelink.UcieTLParams(
               address = 0x208000,
               managerWhere = SBUS,
               numLanes = 16,
-              includeDefaultModels = true
+              includeDefaultModels = true,
+              moduleId = 1,
+              clkDistLayout =
+                if (sim) edu.berkeley.cs.uciedigital.phy.macros.clocking.ClkDistLayout.Behavioral
+                else edu.berkeley.cs.uciedigital.phy.macros.clocking.ClkDistLayout.Buffered("r90")
             ))
         ))) ++
         new WithIrisUncore(sim)

src/UcieComplex.scala

@@ -2,9 +2,11 @@ package edu.berkeley.cs.iris
 
 import org.chipsalliance.cde.config.Parameters
 import freechips.rocketchip.diplomacy._
-import freechips.rocketchip.subsystem.TLBusWrapperLocation
-import edu.berkeley.cs.uciedigital.tilelink.UcieTLParams
+import freechips.rocketchip.subsystem.{BaseSubsystem, TLBusWrapperLocation}
+import freechips.rocketchip.tilelink.{TLBuffer, TLWidthWidget}
+import edu.berkeley.cs.uciedigital.tilelink.{UcieChipletLink, UcieTLParams}
 import testchipip.soc.{
+  CanHaveChipletRouting,
   ChipletLinkParams,
   ChipletLinkWrapper,
   ChipletLinkWrapperInstantiationLike,
@@ -81,3 +83,40 @@ object UciePort {
     .map(_.ports.collect { case UciePort(u) => u })
     .getOrElse(Nil)
 }
+
+/** Attaches the part of each UCIe link that the chiplet router can't.
+  *
+  * The router gives every port one control node, coupled to its control bus,
+  * and one clock. A UCIe link has two register blocks: the main one, on the
+  * clock its PHY makes, and the clock register block, which sets that clock up
+  * and so runs on the chip's. Only the main one goes through the router. This
+  * couples the clock register block to the same bus the same way, and clocks it
+  * from that bus, so its register port needs no crossing.
+  *
+  * The links are found by walking the subsystem's children, since the router
+  * keeps its ports to itself. Extending [[CanHaveChipletRouting]] makes sure
+  * they exist by the time this runs.
+  */
+trait CanHaveUcieClkRegs extends CanHaveChipletRouting { this: BaseSubsystem =>
+  p(ChipletRoutingKey).foreach { params =>
+    def links(lm: LazyModule): Seq[UcieChipletLink] =
+      lm.getChildren.reverse.flatMap {
+        case link: UcieChipletLink => Seq(link)
+        case child                 => links(child)
+      }
+    val ucieLinks = links(this)
+    require(
+      ucieLinks.size == UciePort.all(p).size,
+      s"found ${ucieLinks.size} UCIe links under the subsystem, but the " +
+        s"chiplet router has ${UciePort.all(p).size} UCIe ports"
+    )
+
+    val cbus = locateTLBusWrapper(params.controlBusWhere)
+    ucieLinks.foreach { link =>
+      link.ucie.chipDigitalClockNode := cbus.fixedClockNode
+      cbus.coupleTo(s"${link.name}_clk_control") {
+        link.ucie.clkRegNode := TLWidthWidget(cbus.beatBytes) := TLBuffer() := _
+      }
+    }
+  }
+}