diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad1f93a..d99bc9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ name: build-test-release # # detect (which images changed) # -> build-test (only affected images x arch; per-image self-test) +# -> chipdb (only when pnr-xilinx was rebuilt: a chipdb is only valid for that build) # -> dfhdl-gate (DFHDL latest release + main, against the freshly built images) # -> publish (only on a version tag or manual dispatch, and only if the gate passed) # @@ -61,13 +62,6 @@ jobs: matrix: image: ${{ fromJson(needs.detect.outputs.images) }} arch: [ linux-x64, linux-arm64 ] - exclude: - # TEMPORARY: xezim 0.9.8 does not compile on aarch64 (hardcoded *const i8 - # where c_char is u8) — see https://github.com/aionhw/xezim/issues/105. - # Drop this exclude once upstream fixes it and the XEZIM_* pins are bumped; - # until then sim-xezim releases as linux-x64 only. - - image: sim-xezim - arch: linux-arm64 include: - arch: linux-x64 runs-on: ubuntu-24.04 @@ -108,6 +102,49 @@ jobs: dftools-${{ matrix.image }}-${{ matrix.arch }}.MANIFEST.txt retention-days: 7 + # The nextpnr-xilinx chip databases are a pure function of NEXTPNR_XILINX_REV + PRJXRAY_DB_REV, + # i.e. of the pnr-xilinx image, so they are regenerated exactly when that image is rebuilt and + # never otherwise. Carrying the previous lockfile's chipdbs forward past a nextpnr-xilinx bump + # would be silently wrong: nextpnr asserts that a chipdb's interned IdString table matches its + # own constids.inc and refuses one built by another revision. + # A publish that does not rebuild pnr-xilinx skips this job and keeps the prior chipdb entries, + # which is correct precisely because the revisions did not move. + chipdb: + needs: [ detect, build-test ] + if: contains(needs.detect.outputs.images, 'pnr-xilinx') + runs-on: ubuntu-24.04 + # The parts published today take about a minute each (xc7a35t 46 s, xc7a100t 76 s, measured + # with the image's plain CPython), but upstream reports hours on the large zynq/virtex + # footprints, so the cap is set well under GitHub's 6 h hosted-runner limit rather than tight. + timeout-minutes: 350 + steps: + - uses: actions/checkout@v7 + # The .bba intermediate is ~0.5 GB for the parts published today, tens of GB on the largest. + - name: Free up disk space + uses: jlumbroso/free-disk-space@main + with: { tool-cache: true, large-packages: true, docker-images: true } + - name: Setup Apptainer + run: | + set -eux + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt-get update + sudo apt-get install -y apptainer + - name: Setup JDK + Scala CLI + uses: VirtusLab/scala-cli-setup@v1 + with: { jvm: 'temurin:21' } + # A chipdb is byte-identical on every platform, so one arch generates the set for both. + - uses: actions/download-artifact@v8 + with: { name: dftools-pnr-xilinx-linux-x64, path: dist } + - name: Generate chip databases + run: scala-cli run scripts/chipdb.sc -- dist/dftools-pnr-xilinx-linux-x64.sif chipdb + - uses: actions/upload-artifact@v7 + with: + name: dftools-chipdbs + path: | + chipdb/dftools-chipdb-*.bin.gz + chipdb/dftools-chipdb-*.bin.sha256 + retention-days: 7 + dfhdl-gate: needs: build-test strategy: @@ -159,8 +196,13 @@ jobs: # so freshly built images can be released and used to debug. Publish depends only on a # successful build-test; the dfhdl-gate still runs for visibility but does not block. # Restore `needs: dfhdl-gate` once the gate is green. - needs: build-test - if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.publish) + needs: [ build-test, chipdb ] + # `chipdb` is skipped whenever pnr-xilinx was not rebuilt, and a skipped dependency would + # otherwise skip this job too, so its result is checked explicitly. + if: | + !cancelled() && needs.build-test.result == 'success' + && (needs.chipdb.result == 'success' || needs.chipdb.result == 'skipped') + && (startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.publish)) runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 @@ -173,9 +215,10 @@ jobs: - name: Setup JDK + Scala CLI uses: VirtusLab/scala-cli-setup@v1 with: { jvm: 'temurin:21' } - # Rename the built sifs to immutable, content-addressed asset names and (re)generate the - # lockfile mapping each image+arch to its sha256 + asset. Seeded from the prior lockfile on - # the tag, so a partial re-publish carries unchanged images forward. + # Rename the built sifs and chip databases to immutable, content-addressed asset names and + # (re)generate the lockfile mapping each image+arch (and each chipdb part) to its sha256 + + # asset. Seeded from the prior lockfile on the tag, so a partial re-publish carries unchanged + # images (and the chipdb set, when this run did not regenerate it) forward. - name: Build lockfile + immutable asset names run: scala-cli run scripts/lockfile.sc -- ${{ steps.ver.outputs.tag }} dist # Attach the immutable per-image sifs + the lockfile to the release tag. Immutable names never @@ -190,4 +233,6 @@ jobs: dist/*.sif dist/*.sif.sha256 dist/*.MANIFEST.txt + dist/dftools-chipdb-*.bin.gz + dist/dftools-chipdb-*.bin.sha256 dist/dftools.lock.json diff --git a/README.md b/README.md index 7e9ca5e..27d3b43 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,11 @@ rebuilds the smallest possible image: | **synth-verilog** | yosys (in-tree slang SV frontend), eqy | no VHDL/LLVM — stays lean | | **synth-vhdl** | yosys + ghdl (frontend) + ghdl-yosys-plugin | `yosys -m ghdl`; carries LLVM | | **pnr** | nextpnr-ecp5, nextpnr-himbaechel, ecppack, gowin_pack | consumes yosys JSON | +| **pnr-xilinx** | nextpnr-xilinx, bbasm, fasm2frames, xc7frames2bit, prjxray-db | openXC7; Xilinx 7-series only⁴ | | **sim-llvm** | nvc + ghdl (simulator) | the two VHDL sims share one LLVM | | **sim-verilator** | verilator (+ g++/make/perl) | keeps a C++ build env at runtime¹ | | **sim-iverilog** | iverilog, vvp | small, self-contained | -| **sim-xezim** | xezim | Rust SV simulator; built in lockstep with xezim-core; linux-x64 only until [aionhw/xezim#105](https://github.com/aionhw/xezim/issues/105) | +| **sim-xezim** | xezim | Rust SV simulator; built in lockstep with xezim-core | | **wavegen** | surfer, gtkwave (+ vcd2fst/fst2vcd/… converters), wavecrux³ | GUI; X11-forwarded | | **program** | openFPGALoader | small | | **hmi** | ffmpeg, fpga-isv | media + interactive-sim viewer GUI (X11); single-stage² | @@ -47,6 +48,15 @@ viewers alone. Revisit both (and the pin style) when the canonical [Ferrite-Engineering/wavecrux](https://github.com/Ferrite-Engineering/wavecrux) source repo opens post-beta. +⁴ openXC7 is a *fork* of nextpnr (`openXC7/nextpnr-xilinx`, built with `-DARCH=xilinx`), not an +arch of the YosysHQ tree in **pnr**, so it cannot share that image; it also carries a ~500 MB +prjxray bit database that ECP5/Gowin users should not pay for. Keeping them apart means a Trellis +or Apicula pin bump never rebuilds this image, and vice versa. The per-part chip databases nextpnr +places against are **not** in the image (90-670 MB each): they are generated by +[`scripts/chipdb.sc`](scripts/chipdb.sc) and published as their own assets, see below. +`NEXTPNR_XILINX_REV` must stay at or after `6680df93`: below it `create_clock` does not propagate +past the input BUFG and every build silently reports "PASS at 12.00 MHz". + **ghdl appears in two images on purpose**: the *synthesis frontend* ghdl (`synth-vhdl`, plugin-ABI-bound to yosys) and the *simulator* ghdl (`sim-llvm`) are pinned independently (`GHDL_SYNTH_REV` / `GHDL_SIM_REV`), so a simulator bump never @@ -65,7 +75,8 @@ Diamond, Gowin Designer, QuestaSim) are out of scope — used only via | [`scripts/build.sc`](scripts/build.sc) | `build.sc [dest.sif]` — Scalapptainer build driver | | [`scripts/test.sc`](scripts/test.sc) | `test.sc probe ` / `test.sc dfhdl ` | | [`scripts/affected-images.sh`](scripts/affected-images.sh) | maps a diff to the affected image(s) | -| [`scripts/lockfile.sc`](scripts/lockfile.sc) | rename sifs to immutable names + (re)generate `dftools.lock.json` | +| [`scripts/chipdb.sc`](scripts/chipdb.sc) | generate the nextpnr-xilinx chip databases from a built `pnr-xilinx` image | +| [`scripts/lockfile.sc`](scripts/lockfile.sc) | rename sifs and chipdbs to immutable names + (re)generate `dftools.lock.json` | | [`.github/workflows/release.yml`](.github/workflows/release.yml) | detect → build+test → DFHDL gate → publish | ## How each image stays minimal @@ -114,9 +125,17 @@ sha256 of its sif and the asset carrying those bytes: ```json { "tag": "v0.2.0", "images": { "sim-verilator": { "linux-x64": { "sha256": "ab12…", - "asset": "dftools-sim-verilator-linux-x64-ab12….sif" } } } } + "asset": "dftools-sim-verilator-linux-x64-ab12….sif" } } }, + "chipdbs": { "xc7a100tcsg324": { "sha256": "cd34…", + "asset": "dftools-chipdb-xc7a100tcsg324-cd34….bin.gz" } } } ``` +The `chipdbs` section carries the nextpnr-xilinx chip databases the same way, with two differences: +a chipdb is byte-identical on every platform, so it is not keyed by arch, and it is published +gzipped while the recorded digest is of the decompressed bytes, which is what DFHDL caches and +mmaps. A chipdb is only valid for the `nextpnr-xilinx` build that produced it, so the set is +regenerated exactly when the **pnr-xilinx** image is, and carried forward untouched otherwise. + DFHDL bundles this lockfile at build time and resolves/caches each image **by sha256**, not by tag — so bumping the tag re-downloads only the image(s) whose digest actually changed, and the rest of the set is a cache hit. Only rebuilt images are refreshed on a re-publish; the diff --git a/images/pnr-xilinx.def b/images/pnr-xilinx.def new file mode 100644 index 0000000..baab150 --- /dev/null +++ b/images/pnr-xilinx.def @@ -0,0 +1,137 @@ +# DFTools image: pnr-xilinx +# openXC7 place & route + bitstream generation for Xilinx/AMD 7-series. +# Consumes a Yosys netlist JSON (produced by the synth-* images) plus XDC constraints over the +# mounted workdir, and emits FASM -> frames -> .bit. +# +# Deliberately NOT part of the `pnr` image: nextpnr-xilinx is a separate fork of nextpnr +# (openXC7/nextpnr-xilinx, built with -DARCH=xilinx), not an arch of the YosysHQ tree, and it +# carries a ~500 MB prjxray bit database that ECP5/Gowin users should not pay for. Keeping the +# two apart also means a Trellis or Apicula pin bump never rebuilds this image. +# +# The chipdb (the memory-mapped device database nextpnr-xilinx actually places against) is NOT +# in this image: it is per-part, 90-670 MB each, and takes hours to generate. It is built by +# scripts/chipdb.sc into a separate set of release assets and resolved on demand by DFHDL. + +Bootstrap: docker +From: ubuntu:24.04 +Stage: build + +%files + pins.env /opt/pins.env + build/common.sh /opt/common.sh + build/strip-runtime.sh /opt/strip-runtime.sh + +%post + set -eux + export DEBIAN_FRONTEND=noninteractive + export PREFIX=/opt/dftools + . /opt/pins.env + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates git pkg-config \ + build-essential cmake ninja-build \ + libboost-all-dev libeigen3-dev zlib1g-dev uuid-dev \ + python3 python3-dev python3-venv python3-pip + rm -rf /var/lib/apt/lists/* + . /opt/common.sh + py_venv + + # -- nextpnr-xilinx (openXC7 fork): place & route + FASM writer, plus the bba assembler ---- + # `bbasm` is built but has no install rule upstream, so it is copied by hand. The python + # exporter, constids.inc and the site metadata are installed alongside because they are the + # chipdb generator's inputs (see scripts/chipdb.sc), which runs from this image. + clone https://github.com/openXC7/nextpnr-xilinx "$NEXTPNR_XILINX_REV" nextpnr-xilinx --recursive + ( cd /src/nextpnr-xilinx \ + && cmake -B build -G Ninja \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DARCH=xilinx \ + -DUSE_OPENMP=ON \ + -DBUILD_GUI=OFF -DCMAKE_BUILD_TYPE=Release \ + && ninja -C build && ninja -C build install \ + && install -m755 build/bbasm $PREFIX/bin/bbasm ) + mkdir -p $PREFIX/share/nextpnr-xilinx/external + cp /src/nextpnr-xilinx/xilinx/constids.inc $PREFIX/share/nextpnr-xilinx/constids.inc + cp -r /src/nextpnr-xilinx/xilinx/python $PREFIX/share/nextpnr-xilinx/python + cp -r /src/nextpnr-xilinx/xilinx/external/nextpnr-xilinx-meta \ + $PREFIX/share/nextpnr-xilinx/external/nextpnr-xilinx-meta + + # -- prjxray: FASM -> frames -> bitstream ------------------------------------------------ + # The C++ tools (xc7frames2bit, xc7patch, bitread, ...) link their third-party deps (absl, + # gflags, yaml-cpp) statically, so the final stage needs no extra runtime libraries for them. + clone https://github.com/openXC7/prjxray "$PRJXRAY_REV" prjxray --recursive + ( cd /src/prjxray \ + && cmake -B build -G Ninja \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DPRJXRAY_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + && ninja -C build && ninja -C build install ) + # The python side: the `prjxray` package (database/FASM assembler) and `fasm` (the parser). + # `fasm2frames` is a script under utils/ that upstream's setup.py declares as a console entry + # point but never packages, so it is installed by hand with the venv interpreter's shebang. + pip install --no-cache-dir fasm + pip install --no-cache-dir /src/prjxray + install -m755 /src/prjxray/utils/fasm2frames.py $PREFIX/pyenv/bin/fasm2frames + sed -i "1s|.*|#!$PREFIX/pyenv/bin/python3|" $PREFIX/pyenv/bin/fasm2frames + + # -- prjxray-db: the bit database --------------------------------------------------------- + # Pinned independently of the nextpnr-xilinx submodule so the chipdb inputs are explicit. + # Only families that nextpnr-xilinx also has site metadata for are kept: without metadata the + # bba exporter cannot build a chipdb for them, so the rest is dead weight (hundreds of MB). + clone https://github.com/openXC7/prjxray-db "$PRJXRAY_DB_REV" prjxray-db + mkdir -p $PREFIX/share/prjxray-db + for fam in $(ls $PREFIX/share/nextpnr-xilinx/external/nextpnr-xilinx-meta); do + [ -d "/src/prjxray-db/$fam" ] || continue + cp -r "/src/prjxray-db/$fam" "$PREFIX/share/prjxray-db/$fam" + done + cp /src/prjxray-db/LICENSE $PREFIX/share/prjxray-db/LICENSE 2>/dev/null || true + # Let bbaexport.py's own `