feat(submit): pack design + pins.lock + manifest into a single bundle.zip

chipflow/platform/silicon_step.py

@@ -3,15 +3,17 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
 import inspect
+import io
 import json
 import logging
 import os
 import requests
-import shutil
 import subprocess
 import sys
 import urllib3
 import webbrowser
+import zipfile
+from pathlib import Path
 from pprint import pformat
 
 
@@ -30,6 +32,54 @@
 logger = logging.getLogger(__name__)
 
 
+def _build_bundle_zip(
+    rtlil_path, config: str, project_name: str, process: str, package: str
+) -> bytes:
+    """Pack the submission into a single zip with a manifest.
+
+    Layout::
+
+        manifest.json
+        <rtlil filename>     # e.g. "top.il", taken from rtlil_path
+        pins.lock            # the pinlock JSON
+
+    ``project_name`` / ``process`` / ``package`` come from chipflow.toml
+    (``[chipflow] project_name``, ``[chipflow.silicon] process``,
+    ``[chipflow.silicon] package``). Consumers (logs, dashboards, the
+    backend's working directory naming and PDK / package selection) use
+    these to identify and route the design without re-parsing the pinlock.
+
+    The manifest is the only contract: consumers locate the design and
+    pinlock payloads via ``manifest["design_file"]`` and
+    ``manifest["pins_lock_file"]``. Keys naming a file inside the
+    archive carry a ``_file`` suffix so they're distinguishable from
+    plain value keys (``version``, ``project``, ``process``,
+    ``package``); the value is a zip-relative path. ``design_file`` is
+    named in terms of role rather than format so the same key can carry
+    rtlil today, or another intermediate (Verilog, FIRRTL) tomorrow,
+    without renaming. Future additions (e.g. macro folders) extend the
+    manifest without changing this function's signature on the wire.
+    """
+    design_arc = Path(rtlil_path).name
+    pins_lock_arc = "pins.lock"
+    manifest = {
+        "version": "1",
+        "project": project_name,
+        "process": process,
+        "package": package,
+        "design_file": design_arc,
+        "pins_lock_file": pins_lock_arc,
+    }
+    manifest_bytes = (json.dumps(manifest, indent=2) + "\n").encode("utf-8")
+
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, mode="w", compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.writestr("manifest.json", manifest_bytes)
+        zf.writestr(pins_lock_arc, config)
+        zf.write(str(rtlil_path), arcname=design_arc)
+    return buf.getvalue()
+
+
 def halo_logging(closure):
     class ClosureStreamHandler(logging.StreamHandler):
         def emit(self, record):
@@ -181,16 +231,19 @@ def submit(self, rtlil_path, args):
             pinlock = load_pinlock()
             config = pinlock.model_dump_json(indent=2)
 
+            bundle_bytes = _build_bundle_zip(
+                rtlil_path, config,
+                self.config.chipflow.project_name,
+                self.config.chipflow.silicon.process.value,
+                self.config.chipflow.silicon.package)
+
             if args.dry_run:
                 sp.succeed(f"✅ Design `{data['projectId']}:{data['name']}` ready for submission to ChipFlow cloud!")
                 logger.debug(f"data=\n{json.dumps(data, indent=2)}")
                 logger.debug(f"files['config']=\n{config}")
-                shutil.copyfile(rtlil_path, 'rtlil')
-                with open("rtlil", 'w') as f:
-                    json.dump(data, f)
-                with open("config", 'w') as f:
-                    f.write(config)
-                sp.info("Compiled design and configuration can be found in in `rtlil` and `config`")
+                bundle_path = Path(rtlil_path).parent / "bundle.zip"
+                bundle_path.write_bytes(bundle_bytes)
+                sp.info(f"Compiled submission written to `{bundle_path}` (manifest.json + rtlil + pins.lock)")
                 return
 
             def network_err(e):
@@ -217,8 +270,7 @@ def network_err(e):
                     auth=("", self._chipflow_api_key),
                     data=data,
                     files={
-                        "rtlil": open(rtlil_path, "rb"),
-                        "config": config,
+                        "bundle": ("bundle.zip", bundle_bytes, "application/zip"),
                     },
                     allow_redirects=False
                     )

tests/test_silicon_submit.py

@@ -1,13 +1,16 @@
 # SPDX-License-Identifier: BSD-2-Clause
 
+import io
+import json
 import unittest
+import zipfile
 from unittest import mock
 from argparse import Namespace
 from pathlib import Path
 import tempfile
 import os
 
-from chipflow.platform.silicon_step import SiliconStep
+from chipflow.platform.silicon_step import SiliconStep, _build_bundle_zip
 
 
 class TestSiliconSubmitBrowserPrompt(unittest.TestCase):
@@ -47,16 +50,17 @@ def test_browser_prompt_yes(self, mock_subprocess, mock_isatty, mock_input, mock
         # Create a mock SiliconStep instance
         with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'):
             config = mock.MagicMock()
-            config.chipflow.silicon = True
+            config.chipflow.silicon.process.value = 'sky130'
+            config.chipflow.silicon.package = 'cf20'
             config.chipflow.project_name = 'test_project'
             step = SiliconStep(config)
             step._build_url = "https://build.chipflow.com/build/test123"
             step.platform._ports = {}
 
             # Mock the submit method dependencies
-            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'):
-                with mock.patch('builtins.open', mock.mock_open(read_data=b'')):
-                    with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
+            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \
+                    mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'):
+                with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
                         # Mock successful submission
                         mock_response = mock.MagicMock()
                         mock_response.status_code = 200
@@ -96,16 +100,17 @@ def test_browser_prompt_no(self, mock_subprocess, mock_isatty, mock_input, mock_
         # Create a mock SiliconStep instance
         with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'):
             config = mock.MagicMock()
-            config.chipflow.silicon = True
+            config.chipflow.silicon.process.value = 'sky130'
+            config.chipflow.silicon.package = 'cf20'
             config.chipflow.project_name = 'test_project'
             step = SiliconStep(config)
             step._build_url = "https://build.chipflow.com/build/test123"
             step.platform._ports = {}
 
             # Mock the submit method dependencies
-            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'):
-                with mock.patch('builtins.open', mock.mock_open(read_data=b'')):
-                    with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
+            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \
+                    mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'):
+                with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
                         # Mock successful submission
                         mock_response = mock.MagicMock()
                         mock_response.status_code = 200
@@ -143,16 +148,17 @@ def test_browser_prompt_not_tty(self, mock_subprocess, mock_isatty, mock_input,
         # Create a mock SiliconStep instance
         with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'):
             config = mock.MagicMock()
-            config.chipflow.silicon = True
+            config.chipflow.silicon.process.value = 'sky130'
+            config.chipflow.silicon.package = 'cf20'
             config.chipflow.project_name = 'test_project'
             step = SiliconStep(config)
             step._build_url = "https://build.chipflow.com/build/test123"
             step.platform._ports = {}
 
             # Mock the submit method dependencies
-            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'):
-                with mock.patch('builtins.open', mock.mock_open(read_data=b'')):
-                    with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
+            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \
+                    mock.patch('chipflow.platform.silicon_step._build_bundle_zip', return_value=b'fake-bundle'):
+                with mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post:
                         # Mock successful submission
                         mock_response = mock.MagicMock()
                         mock_response.status_code = 200
@@ -175,5 +181,82 @@ def test_browser_prompt_not_tty(self, mock_subprocess, mock_isatty, mock_input,
                                 mock_webbrowser.assert_not_called()
 
 
+class TestBuildBundleZip(unittest.TestCase):
+    """Tests for the _build_bundle_zip helper."""
+
+    def test_manifest_and_layout(self):
+        with tempfile.TemporaryDirectory() as td:
+            rtlil_path = Path(td) / "top.il"
+            rtlil_path.write_text("module top(); endmodule\n")
+            config = '{"pins": []}'
+
+            blob = _build_bundle_zip(rtlil_path, config, "my_project", "sky130", "cf20")
+
+            with zipfile.ZipFile(io.BytesIO(blob)) as zf:
+                names = set(zf.namelist())
+                self.assertEqual(names, {"manifest.json", "top.il", "pins.lock"})
+
+                manifest = json.loads(zf.read("manifest.json"))
+                self.assertEqual(manifest["version"], "1")
+                self.assertEqual(manifest["project"], "my_project")
+                self.assertEqual(manifest["process"], "sky130")
+                self.assertEqual(manifest["package"], "cf20")
+                self.assertEqual(manifest["design_file"], "top.il")
+                self.assertEqual(manifest["pins_lock_file"], "pins.lock")
+
+                self.assertEqual(zf.read("top.il").decode(), "module top(); endmodule\n")
+                self.assertEqual(zf.read("pins.lock").decode(), config)
+
+    def test_uses_real_rtlil_filename(self):
+        """Bundle preserves the source rtlil filename (not a fixed string)."""
+        with tempfile.TemporaryDirectory() as td:
+            rtlil_path = Path(td) / "weird_name.rtlil"
+            rtlil_path.write_text("x")
+            blob = _build_bundle_zip(rtlil_path, "{}", "p", "sky130", "cf20")
+            with zipfile.ZipFile(io.BytesIO(blob)) as zf:
+                self.assertIn("weird_name.rtlil", zf.namelist())
+                manifest = json.loads(zf.read("manifest.json"))
+                self.assertEqual(manifest["design_file"], "weird_name.rtlil")
+
+
+class TestSiliconSubmitBundlePost(unittest.TestCase):
+    """The submit() path posts a single 'bundle' multipart part."""
+
+    @mock.patch('chipflow.packaging.load_pinlock')
+    @mock.patch('chipflow.platform.silicon_step.subprocess.check_output')
+    def test_submit_sends_single_bundle_part(self, mock_subprocess, mock_load_pinlock):
+        mock_subprocess.return_value = 'test123\n'
+        mock_pinlock = mock.MagicMock()
+        mock_pinlock.model_dump_json.return_value = '{}'
+        mock_load_pinlock.return_value = mock_pinlock
+
+        with mock.patch('chipflow.platform.silicon_step.SiliconPlatform'):
+            config = mock.MagicMock()
+            config.chipflow.silicon.process.value = 'sky130'
+            config.chipflow.silicon.package = 'cf20'
+            config.chipflow.project_name = 'test_project'
+            step = SiliconStep(config)
+            step.platform._ports = {}
+
+            with mock.patch.object(step, 'prepare', return_value='/tmp/test.il'), \
+                    mock.patch('chipflow.platform.silicon_step._build_bundle_zip',
+                               return_value=b'fake-bundle-bytes'), \
+                    mock.patch('chipflow.platform.silicon_step.requests.post') as mock_post, \
+                    mock.patch('chipflow.platform.silicon_step.get_api_key', return_value='k'), \
+                    mock.patch('chipflow.platform.silicon_step.exit'), \
+                    mock.patch('sys.stdout.isatty', return_value=False):
+                mock_post.return_value = mock.MagicMock(
+                    status_code=200, json=lambda: {'build_id': 'b1'})
+                step._chipflow_api_key = 'k'
+                step.submit('/tmp/test.il', Namespace(dry_run=False, wait=False))
+
+            files = mock_post.call_args.kwargs["files"]
+            self.assertEqual(set(files.keys()), {"bundle"})
+            filename, payload, content_type = files["bundle"]
+            self.assertEqual(filename, "bundle.zip")
+            self.assertEqual(payload, b'fake-bundle-bytes')
+            self.assertEqual(content_type, "application/zip")
+
+
 if __name__ == "__main__":
     unittest.main()