diff --git a/chipflow/platform/silicon_step.py b/chipflow/platform/silicon_step.py index 8ab5c354..6bd15e40 100644 --- a/chipflow/platform/silicon_step.py +++ b/chipflow/platform/silicon_step.py @@ -3,15 +3,17 @@ # SPDX-License-Identifier: BSD-2-Clause import inspect +import io import json import logging import os import requests -import shutil import subprocess import sys import urllib3 import webbrowser +import zipfile +from pathlib import Path from pprint import pformat @@ -30,6 +32,54 @@ logger = logging.getLogger(__name__) +def _build_bundle_zip( + rtlil_path, config: str, project_name: str, process: str, package: str +) -> bytes: + """Pack the submission into a single zip with a manifest. + + Layout:: + + manifest.json + # e.g. "top.il", taken from rtlil_path + pins.lock # the pinlock JSON + + ``project_name`` / ``process`` / ``package`` come from chipflow.toml + (``[chipflow] project_name``, ``[chipflow.silicon] process``, + ``[chipflow.silicon] package``). Consumers (logs, dashboards, the + backend's working directory naming and PDK / package selection) use + these to identify and route the design without re-parsing the pinlock. + + The manifest is the only contract: consumers locate the design and + pinlock payloads via ``manifest["design_file"]`` and + ``manifest["pins_lock_file"]``. Keys naming a file inside the + archive carry a ``_file`` suffix so they're distinguishable from + plain value keys (``version``, ``project``, ``process``, + ``package``); the value is a zip-relative path. ``design_file`` is + named in terms of role rather than format so the same key can carry + rtlil today, or another intermediate (Verilog, FIRRTL) tomorrow, + without renaming. Future additions (e.g. macro folders) extend the + manifest without changing this function's signature on the wire. + """ + design_arc = Path(rtlil_path).name + pins_lock_arc = "pins.lock" + manifest = { + "version": "1", + "project": project_name, + "process": process, + "package": package, + "design_file": design_arc, + "pins_lock_file": pins_lock_arc, + } + manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode("utf-8") + + buf = io.BytesIO() + with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr("manifest.json", manifest_bytes) + zf.writestr(pins_lock_arc, config) + zf.write(str(rtlil_path), arcname=design_arc) + return buf.getvalue() + + def halo_logging(closure): class ClosureStreamHandler(logging.StreamHandler): def emit(self, record): @@ -181,16 +231,19 @@ def submit(self, rtlil_path, args): pinlock = load_pinlock() config = pinlock.model_dump_json(indent=2) + bundle_bytes = _build_bundle_zip( + rtlil_path, config, + self.config.chipflow.project_name, + self.config.chipflow.silicon.process.value, + self.config.chipflow.silicon.package) + if args.dry_run: sp.succeed(f"✅ Design `{data['projectId']}:{data['name']}` ready for submission to ChipFlow cloud!") logger.debug(f"data=\n{json.dumps(data, indent=2)}") logger.debug(f"files['config']=\n{config}") - shutil.copyfile(rtlil_path, 'rtlil') - with open("rtlil", 'w') as f: - json.dump(data, f) - with open("config", 'w') as f: - f.write(config) - sp.info("Compiled design and configuration can be found in in `rtlil` and `config`") + bundle_path = Path(rtlil_path).parent / "bundle.zip" + bundle_path.write_bytes(bundle_bytes) + sp.info(f"Compiled submission written to `{bundle_path}` (manifest.json + rtlil + pins.lock)") return def network_err(e): @@ -217,8 +270,7 @@ def network_err(e): auth=("", self._chipflow_api_key), data=data, files={ - "rtlil": open(rtlil_path, "rb"), - "config": config, + "bundle": ("bundle.zip", bundle_bytes, "application/zip"), }, allow_redirects=False ) diff --git a/tests/test_silicon_submit.py b/tests/test_silicon_submit.py index 83c10a18..fabf0853 100644 --- a/tests/test_silicon_submit.py +++ b/tests/test_silicon_submit.py @@ -1,13 +1,16 @@ # SPDX-License-Identifier: BSD-2-Clause +import io +import json import unittest +import zipfile from unittest import mock from argparse import Namespace from pathlib import Path import tempfile import os -from chipflow.platform.silicon_step import SiliconStep +from chipflow.platform.silicon_step import SiliconStep, _build_bundle_zip class TestSiliconSubmitBrowserPrompt(unittest.TestCase): @@ -47,16 +50,17 @@ def test_browser_prompt_yes(self, mock_subprocess, mock_isatty, mock_input, mock # Create a mock SiliconStep instance with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'): config = mock.MagicMock() - config.chipflow.silicon = True + config.chipflow.silicon.process.value = 'sky130' + config.chipflow.silicon.package = 'cf20' config.chipflow.project_name = 'test_project' step = SiliconStep(config) step._build_url = "https://build.chipflow.com/build/test123" step.platform._ports = {} # Mock the submit method dependencies - with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'): - with mock.patch('builtins.open', mock.mock_open(read_data=b'')): - with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: + with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \ + mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'): + with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: # Mock successful submission mock_response = mock.MagicMock() mock_response.status_code = 200 @@ -96,16 +100,17 @@ def test_browser_prompt_no(self, mock_subprocess, mock_isatty, mock_input, mock_ # Create a mock SiliconStep instance with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'): config = mock.MagicMock() - config.chipflow.silicon = True + config.chipflow.silicon.process.value = 'sky130' + config.chipflow.silicon.package = 'cf20' config.chipflow.project_name = 'test_project' step = SiliconStep(config) step._build_url = "https://build.chipflow.com/build/test123" step.platform._ports = {} # Mock the submit method dependencies - with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'): - with mock.patch('builtins.open', mock.mock_open(read_data=b'')): - with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: + with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \ + mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'): + with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: # Mock successful submission mock_response = mock.MagicMock() mock_response.status_code = 200 @@ -143,16 +148,17 @@ def test_browser_prompt_not_tty(self, mock_subprocess, mock_isatty, mock_input, # Create a mock SiliconStep instance with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'): config = mock.MagicMock() - config.chipflow.silicon = True + config.chipflow.silicon.process.value = 'sky130' + config.chipflow.silicon.package = 'cf20' config.chipflow.project_name = 'test_project' step = SiliconStep(config) step._build_url = "https://build.chipflow.com/build/test123" step.platform._ports = {} # Mock the submit method dependencies - with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'): - with mock.patch('builtins.open', mock.mock_open(read_data=b'')): - with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: + with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \ + mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'): + with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post: # Mock successful submission mock_response = mock.MagicMock() mock_response.status_code = 200 @@ -175,5 +181,82 @@ def test_browser_prompt_not_tty(self, mock_subprocess, mock_isatty, mock_input, mock_webbrowser.assert_not_called() +class TestBuildBundleZip(unittest.TestCase): + """Tests for the _build_bundle_zip helper.""" + + def test_manifest_and_layout(self): + with tempfile.TemporaryDirectory() as td: + rtlil_path = Path(td) / "top.il" + rtlil_path.write_text("module top(); endmodule\n") + config = '{"pins": []}' + + blob = _build_bundle_zip(rtlil_path, config, "my_project", "sky130", "cf20") + + with zipfile.ZipFile(io.BytesIO(blob)) as zf: + names = set(zf.namelist()) + self.assertEqual(names, {"manifest.json", "top.il", "pins.lock"}) + + manifest = json.loads(zf.read("manifest.json")) + self.assertEqual(manifest["version"], "1") + self.assertEqual(manifest["project"], "my_project") + self.assertEqual(manifest["process"], "sky130") + self.assertEqual(manifest["package"], "cf20") + self.assertEqual(manifest["design_file"], "top.il") + self.assertEqual(manifest["pins_lock_file"], "pins.lock") + + self.assertEqual(zf.read("top.il").decode(), "module top(); endmodule\n") + self.assertEqual(zf.read("pins.lock").decode(), config) + + def test_uses_real_rtlil_filename(self): + """Bundle preserves the source rtlil filename (not a fixed string).""" + with tempfile.TemporaryDirectory() as td: + rtlil_path = Path(td) / "weird_name.rtlil" + rtlil_path.write_text("x") + blob = _build_bundle_zip(rtlil_path, "{}", "p", "sky130", "cf20") + with zipfile.ZipFile(io.BytesIO(blob)) as zf: + self.assertIn("weird_name.rtlil", zf.namelist()) + manifest = json.loads(zf.read("manifest.json")) + self.assertEqual(manifest["design_file"], "weird_name.rtlil") + + +class TestSiliconSubmitBundlePost(unittest.TestCase): + """The submit() path posts a single 'bundle' multipart part.""" + + @mock.patch('chipflow.packaging.load_pinlock') + @mock.patch('chipflow.platform.silicon_step.subprocess.check_output') + def test_submit_sends_single_bundle_part(self, mock_subprocess, mock_load_pinlock): + mock_subprocess.return_value = 'test123\n' + mock_pinlock = mock.MagicMock() + mock_pinlock.model_dump_json.return_value = '{}' + mock_load_pinlock.return_value = mock_pinlock + + with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'): + config = mock.MagicMock() + config.chipflow.silicon.process.value = 'sky130' + config.chipflow.silicon.package = 'cf20' + config.chipflow.project_name = 'test_project' + step = SiliconStep(config) + step.platform._ports = {} + + with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \ + mock.patch('chipflow.platform.silicon_step._build_bundle_zip', + return_value=b'fake-bundle-bytes'), \ + mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post, \ + mock.patch('chipflow.platform.silicon_step.get_api_key', return_value='k'), \ + mock.patch('chipflow.platform.silicon_step.exit'), \ + mock.patch('sys.stdout.isatty', return_value=False): + mock_post.return_value = mock.MagicMock( + status_code=200, json=lambda: {'build_id': 'b1'}) + step._chipflow_api_key = 'k' + step.submit('/tmp/test.il', Namespace(dry_run=False, wait=False)) + + files = mock_post.call_args.kwargs["files"] + self.assertEqual(set(files.keys()), {"bundle"}) + filename, payload, content_type = files["bundle"] + self.assertEqual(filename, "bundle.zip") + self.assertEqual(payload, b'fake-bundle-bytes') + self.assertEqual(content_type, "application/zip") + + if __name__ == "__main__": unittest.main()