diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71ddf96a..7e331233 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ on: - feat/ar1-metadata-fabric-retained-real-feature-terminal-success - feat/ar1-metadata-fabric-retained-real-feature-restart-recovery - feat/ar1-metadata-fabric-protected-real-feature-reexecution-gate + - feat/ar1-metadata-fabric-protected-profile-decision-packet env: PYTHON_VERSION: "3.13" @@ -220,6 +221,9 @@ jobs: - name: Validate protected profile owner decision packet run: python -m data_agent.metadata_fabric_protected_profile_decision_packet validate + - name: Validate Chongqing real-source admission evidence + run: python -m data_agent.chongqing_real_source_admission validate + - name: Validate Active Metadata consumer deployment boundary run: python -m data_agent.active_metadata_consumer_deployment validate @@ -321,6 +325,7 @@ jobs: data_agent/test_metadata_fabric_retained_real_feature_restart_recovery.py \ data_agent/test_metadata_fabric_protected_real_feature_reexecution_gate.py \ data_agent/test_metadata_fabric_protected_profile_decision_packet.py \ + data_agent/test_chongqing_real_source_admission.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_gravitino_identity.py \ diff --git a/data_agent/chongqing_real_source_admission.py b/data_agent/chongqing_real_source_admission.py new file mode 100644 index 00000000..8c631b6f --- /dev/null +++ b/data_agent/chongqing_real_source_admission.py @@ -0,0 +1,1082 @@ +"""Build a fail-closed admission manifest for the full Chongqing sample. + +M3-28 profiles the real planning-institute archive as an AR-2 source without +copying source payloads into the repository. The checked evidence is path-free, +content-addressed, and metadata-only. It never authorizes ingestion, +publication, scheduler submission, or provider mutation. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import re +import stat +import zipfile +import zlib +from collections import Counter +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .uwm.local_planning_zip_audit import scan_local_planning_zip_assets + +EVIDENCE_SCHEMA = "gda.chongqing_real_source_admission.v1" +VALIDATION_SCHEMA = "gda.chongqing_real_source_admission_validation.v1" +STATUS = "blocked_pending_source_governance" +SOURCE_ID = "chongqing-planning-institute-sample" + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_EVIDENCE_PATH = REPO_ROOT / "docs/evidence/chongqing-real-source-admission-2026-07-31.json" +RESEARCH_INVENTORY_PATH = ( + REPO_ROOT / "data/uwm_public_proxy/chongqing_central/" + "local_planning_zip_audit_2026_07_05/uwm_local_planning_zip_inventory.csv" +) + +EXPECTED_ARCHIVE_SHA256 = "2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca" +EXPECTED_ARCHIVE_SIZE_BYTES = 468_462_251 +EXPECTED_ARCHIVE_ENTRY_COUNT = 533 +EXPECTED_ARCHIVE_UNCOMPRESSED_SIZE_BYTES = 694_164_379 +EXPECTED_ARCHIVE_SCOPE_ENTRY_COUNT = 532 +EXPECTED_ARCHIVE_SCOPE_SIZE_BYTES = 694_147_946 +EXPECTED_EXTRACTED_FILE_COUNT = 584 +EXPECTED_EXTRACTED_SIZE_BYTES = 700_610_744 +EXPECTED_ARCHIVE_EXACT_MATCH_COUNT = 526 +EXPECTED_ARCHIVE_MODIFIED_COUNT = 6 +EXPECTED_ARCHIVE_MISSING_COUNT = 0 +EXPECTED_EXTRACTED_ADDITIONAL_COUNT = 52 +EXPECTED_EXTRACTED_PAYLOAD_SHA256 = ( + "e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6" +) +EXPECTED_RESEARCH_INVENTORY_SHA256 = ( + "69b4167955f950041988dd174b75ea5376af146ef3e52f815aa57715fd24f70d" +) +EXPECTED_RESEARCH_INVENTORY_ROWS = 16 +EXPECTED_RESEARCH_ASSET_IDS_SHA256 = ( + "c7cb765c2b653dbf5619e2fe1cf027d108a5951b752b6dd4bfd60b4c00d91947" +) +EVIDENCE_FILE_SHA256 = "9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83" + +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +SOURCE_REF_PATTERN = re.compile(rf"^source://{re.escape(SOURCE_ID)}/assets/[a-z0-9][a-z0-9_-]+$") +SENSITIVE_KEY_PATTERN = re.compile( + r"(^|[-_.])(password|passwd|secret|client[-_.]?secret|private[-_.]?key|" + r"access[-_.]?key|access[-_.]?token|refresh[-_.]?token|" + r"authorization[-_.]?header)($|[-_.])", + re.IGNORECASE, +) + +EVIDENCE_INVENTORY = { + "schema", + "status", + "captured_at", + "source_binding", + "research_audit_binding", + "source_groups", + "asset_profiles", + "admission_blockers", + "admission_policy", + "claims", + "evidence_sha256", +} +SOURCE_BINDING_INVENTORY = { + "source_id", + "archive_sha256", + "archive_size_bytes", + "archive_entry_count", + "archive_uncompressed_size_bytes", + "archive_source_scope_entry_count", + "archive_source_scope_size_bytes", + "archive_original_entry_exact_match_count", + "archive_original_entry_modified_count", + "archive_original_entry_missing_count", + "extracted_additional_file_count", + "archive_integrity_verified", + "archive_extracted_entry_multiset_verified", + "extracted_file_count", + "extracted_size_bytes", + "extracted_payload_sha256", + "logical_group_inventory_sha256", + "source_payload_in_repository", + "absolute_source_paths_in_evidence", +} +RESEARCH_AUDIT_INVENTORY = { + "path", + "file_sha256", + "row_count", + "asset_ids_sha256", + "authority", + "admission_authority", +} +SOURCE_GROUP_INVENTORY = { + "source_group_id", + "ordinal", + "file_count", + "size_bytes", + "extension_counts", + "content_manifest_sha256", + "data_domains", + "data_classification", + "license_status", + "admission_status", + "metadata_profiled", + "content_admitted", + "blockers", +} +ASSET_PROFILE_INVENTORY = { + "asset_id", + "source_group_id", + "source_ref", + "asset_kind", + "data_classification", + "admission_status", + "record_metrics", + "spatial_profile", + "schema_fields", + "roles", + "metadata_profiled", + "source_content_in_evidence", + "profile_sha256", +} +CLAIMS = { + "archive_integrity_verified", + "archive_extracted_entry_multiset_verified", + "full_source_metadata_profiled", + "source_governance_approved", + "source_content_admitted", + "source_publication_authorized", + "scheduler_submission_authorized", + "provider_mutation_authorized", + "production_ingestion_verified", + "production_ready", +} +SOURCE_BINDING_BLOCKERS = [ + "source_binding:extraction_derivation_provenance_missing", +] +ADMISSION_POLICY = { + "metadata_profiling_allowed": True, + "content_admission_requires_owner_license_retention_and_access_approval": True, + "restricted_fields_require_privacy_review": True, + "source_payload_copy_to_repository_forbidden": True, + "ci_requires_local_source_payload": False, + "fresh_protected_ingestion_required_after_admission": True, + "local_profile_is_not_production_admission": True, +} + + +SOURCE_GROUP_SPECS: tuple[dict[str, Any], ...] = ( + { + "ordinal": "01", + "source_group_id": "chongqing-dem-2020", + "data_domains": ["elevation", "terrain", "raster"], + "data_classification": "restricted_local_baseline", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "02", + "source_group_id": "chongqing-osm-roads-2021", + "data_domains": ["transport_network", "vector"], + "data_classification": "restricted_local_open_data_copy", + "additional_blockers": ["odbl_attribution_policy_missing"], + }, + { + "ordinal": "03", + "source_group_id": "chongqing-clcd-2020", + "data_domains": ["land_cover", "remote_sensing", "raster"], + "data_classification": "restricted_local_remote_sensing", + "additional_blockers": ["classification_lineage_unverified"], + }, + { + "ordinal": "04", + "source_group_id": "chongqing-central-buildings-2021", + "data_domains": ["buildings", "urban_form", "vector"], + "data_classification": "restricted_local_built_environment", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "05", + "source_group_id": "chongqing-historic-districts", + "data_domains": ["cultural_heritage", "planning_constraints", "vector"], + "data_classification": "restricted_local_cultural_planning", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "07", + "source_group_id": "bishan-planning-materials", + "data_domains": ["land_use", "planning", "documents", "vector", "tables"], + "data_classification": "highly_restricted_planning", + "additional_blockers": ["planning_sensitivity_review_missing"], + }, + { + "ordinal": "08", + "source_group_id": "chongqing-district-population-2021", + "data_domains": ["population", "statistics", "table"], + "data_classification": "restricted_aggregate_population", + "additional_blockers": ["statistics_vintage_review_missing"], + }, + { + "ordinal": "09", + "source_group_id": "gaode-poi-2024", + "data_domains": ["poi", "commercial_location", "vector"], + "data_classification": "highly_restricted_commercial_location", + "additional_blockers": ["contact_field_privacy_review_missing"], + }, + { + "ordinal": "10", + "source_group_id": "baidu-aoi-2024", + "data_domains": ["aoi", "commercial_location", "vector"], + "data_classification": "highly_restricted_commercial_location", + "additional_blockers": ["contact_field_privacy_review_missing"], + }, + { + "ordinal": "11", + "source_group_id": "unicom-commuting-2023", + "data_domains": ["mobility", "population", "aggregate_signaling", "table"], + "data_classification": "highly_restricted_aggregate_mobility", + "additional_blockers": [ + "privacy_impact_assessment_missing", + "grid_geometry_dictionary_missing", + ], + }, + { + "ordinal": "12", + "source_group_id": "baidu-search-index-2023", + "data_domains": ["search_activity", "intercity_flow", "vector"], + "data_classification": "highly_restricted_commercial_activity", + "additional_blockers": ["commercial_terms_review_missing"], + }, +) + +GROUP_BY_ORDINAL = {spec["ordinal"]: spec for spec in SOURCE_GROUP_SPECS} +GROUP_BY_ID = {spec["source_group_id"]: spec for spec in SOURCE_GROUP_SPECS} + +EXPECTED_ASSET_BASELINES: dict[str, dict[str, Any]] = { + "chongqing_osm_roads_2021": { + "source_group_id": "chongqing-osm-roads-2021", + "asset_kind": "vector", + "record_metrics": {"feature_count": 50_366}, + "geometry_type": "LineString", + "crs": "EPSG:4326", + }, + "chongqing_central_buildings_2021": { + "source_group_id": "chongqing-central-buildings-2021", + "asset_kind": "vector", + "record_metrics": {"feature_count": 107_452}, + "geometry_type": "Polygon", + "crs": "EPSG:4326", + }, + "chongqing_historic_districts_local": { + "source_group_id": "chongqing-historic-districts", + "asset_kind": "vector", + "record_metrics": {"feature_count": 20}, + "geometry_type": "Polygon Z", + "crs": "EPSG:4490", + }, + "bishan_land_use_dltb_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 101_657}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4610", + }, + "gaode_poi_2024": { + "source_group_id": "gaode-poi-2024", + "asset_kind": "vector", + "record_metrics": {"feature_count": 1_194_351}, + "geometry_type": "Point", + "crs": "EPSG:4490", + }, + "baidu_aoi_2024": { + "source_group_id": "baidu-aoi-2024", + "asset_kind": "vector", + "record_metrics": {"feature_count": 26_292}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4490", + }, + "baidu_search_index_2023_local": { + "source_group_id": "baidu-search-index-2023", + "asset_kind": "vector", + "record_metrics": {"feature_count": 325}, + "geometry_type": "MultiLineString", + "crs": "EPSG:4490", + }, + "bishan_admin_boundary_cjdcq_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 1_488}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4523+EPSG:5737", + }, + "bishan_admin_boundary_xzq_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 15}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4523+EPSG:5737", + }, + "fulu_village_planning_database_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector_collection", + "record_metrics": { + "feature_count": 8_050, + "layer_count": 28, + "nonempty_layer_count": 20, + }, + "geometry_type": "mixed", + "crs": "mixed_CGCS2000_GK_zone_35_EPSG4523", + }, + "chongqing_district_population_stats_2021_local": { + "source_group_id": "chongqing-district-population-2021", + "asset_kind": "workbook", + "record_metrics": {"row_count": 41, "sheet_count": 1}, + }, + "chongqing_unicom_commuting_2023_local": { + "source_group_id": "unicom-commuting-2023", + "asset_kind": "table", + "record_metrics": {"row_count": 2_120, "column_count": 7}, + }, + "clcd_classification_dictionary_local": { + "source_group_id": "chongqing-clcd-2020", + "asset_kind": "workbook", + "record_metrics": {"row_count": 10, "sheet_count": 1}, + }, + "bishan_land_development_ledger_2019_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "workbook_collection", + "record_metrics": {"row_count": 1_438, "sheet_count": 4}, + }, + "chongqing_dem_80m": { + "source_group_id": "chongqing-dem-2020", + "asset_kind": "raster", + "record_metrics": { + "width": 1_766, + "height": 1_454, + "band_count": 1, + "pixel_count": 2_567_764, + }, + "crs": "EPSG:4490", + }, + "chongqing_clcd_2020": { + "source_group_id": "chongqing-clcd-2020", + "asset_kind": "raster", + "record_metrics": { + "width": 18_579, + "height": 15_082, + "band_count": 1, + "pixel_count": 280_208_478, + }, + "crs": "EPSG:4326", + }, +} + + +class ChongqingRealSourceAdmissionError(RuntimeError): + """The full-source admission snapshot failed closed.""" + + +def canonical_json_fingerprint(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + return hashlib.sha256(payload).hexdigest() + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document is not an object") + return value + + +def _parse_time(value: Any) -> datetime: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError as exc: + raise ChongqingRealSourceAdmissionError("captured_at is not a valid timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ChongqingRealSourceAdmissionError("captured_at must be timezone-aware") + return parsed.astimezone(UTC) + + +def _source_group_blockers(spec: Mapping[str, Any]) -> list[str]: + group_id = str(spec["source_group_id"]) + suffixes = [ + "owner_reference_missing", + "license_terms_unverified", + "retention_policy_missing", + "access_policy_missing", + *[str(item) for item in spec.get("additional_blockers", [])], + ] + return [f"source_group:{group_id}:{suffix}" for suffix in suffixes] + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_mode, + ) + + +def _hash_regular_file(path: Path) -> tuple[int, str, str]: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ChongqingRealSourceAdmissionError("source payload contains a non-regular file") + sha256 = hashlib.sha256() + crc32 = 0 + with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + if _stat_identity(opened) != _stat_identity(before): + raise ChongqingRealSourceAdmissionError( + "source payload file identity changed before hashing" + ) + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + sha256.update(chunk) + crc32 = zlib.crc32(chunk, crc32) + after = os.fstat(stream.fileno()) + if _stat_identity(after) != _stat_identity(before): + raise ChongqingRealSourceAdmissionError( + "source payload file identity changed while hashing" + ) + return before.st_size, sha256.hexdigest(), f"{crc32 & 0xFFFFFFFF:08x}" + + +def _scan_payload_files(source_root: Path) -> list[dict[str, Any]]: + root = source_root.resolve(strict=True) + root_stat = source_root.lstat() + if stat.S_ISLNK(root_stat.st_mode) or not stat.S_ISDIR(root_stat.st_mode): + raise ChongqingRealSourceAdmissionError("source root must be a real directory") + records: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + if path.is_symlink(): + raise ChongqingRealSourceAdmissionError("source payload contains a symbolic link") + if not path.is_file(): + continue + relative_path = path.relative_to(root).as_posix() + ordinal = relative_path.split("/", 1)[0][:2] + spec = GROUP_BY_ORDINAL.get(ordinal) + if spec is None: + raise ChongqingRealSourceAdmissionError( + f"source payload contains an unknown top-level group: {ordinal}" + ) + size, sha256, crc32 = _hash_regular_file(path) + suffix = path.suffix.lower().lstrip(".") or "[no_ext]" + records.append( + { + "relative_path": relative_path, + "source_group_id": spec["source_group_id"], + "size_bytes": size, + "extension": suffix, + "sha256": sha256, + "crc32": crc32, + } + ) + if not records: + raise ChongqingRealSourceAdmissionError("source payload is empty") + return records + + +def _payload_fingerprint(records: list[Mapping[str, Any]]) -> str: + material = [ + { + "relative_path": item["relative_path"], + "size_bytes": item["size_bytes"], + "sha256": item["sha256"], + } + for item in records + ] + return canonical_json_fingerprint(material) + + +def _build_source_groups(records: list[Mapping[str, Any]]) -> list[dict[str, Any]]: + groups: list[dict[str, Any]] = [] + for spec in SOURCE_GROUP_SPECS: + group_records = [ + item for item in records if item.get("source_group_id") == spec["source_group_id"] + ] + if not group_records: + raise ChongqingRealSourceAdmissionError( + f"source group is empty: {spec['source_group_id']}" + ) + manifest = [ + { + "relative_path": item["relative_path"], + "size_bytes": item["size_bytes"], + "sha256": item["sha256"], + } + for item in group_records + ] + extensions = Counter(str(item["extension"]) for item in group_records) + groups.append( + { + "source_group_id": spec["source_group_id"], + "ordinal": spec["ordinal"], + "file_count": len(group_records), + "size_bytes": sum(int(item["size_bytes"]) for item in group_records), + "extension_counts": dict(sorted(extensions.items())), + "content_manifest_sha256": canonical_json_fingerprint(manifest), + "data_domains": list(spec["data_domains"]), + "data_classification": spec["data_classification"], + "license_status": "unverified_restricted", + "admission_status": STATUS, + "metadata_profiled": True, + "content_admitted": False, + "blockers": _source_group_blockers(spec), + } + ) + return groups + + +def _archive_binding( + source_zip: Path, + payload_records: list[Mapping[str, Any]], +) -> dict[str, Any]: + archive_stat = source_zip.lstat() + if stat.S_ISLNK(archive_stat.st_mode) or not stat.S_ISREG(archive_stat.st_mode): + raise ChongqingRealSourceAdmissionError("source archive must be a regular file") + archive_sha256 = _file_sha256(source_zip) + with zipfile.ZipFile(source_zip) as archive: + bad_entry = archive.testzip() + if bad_entry is not None: + raise ChongqingRealSourceAdmissionError("source archive CRC validation failed") + all_entries = [item for item in archive.infolist() if not item.is_dir()] + scoped_entries = [ + (item, relative_path) + for item in archive.infolist() + if not item.is_dir() + and (relative_path := _source_scope_relative_path(item)) is not None + ] + extracted_by_path = {str(item["relative_path"]): item for item in payload_records} + scoped_paths = {relative_path for _, relative_path in scoped_entries} + if len(scoped_paths) != len(scoped_entries): + raise ChongqingRealSourceAdmissionError("source archive scope contains duplicate paths") + exact_match_count = 0 + modified_count = 0 + missing_count = 0 + for item, relative_path in scoped_entries: + extracted = extracted_by_path.get(relative_path) + if extracted is None: + missing_count += 1 + continue + if ( + int(extracted["size_bytes"]) == item.file_size + and str(extracted["crc32"]) == f"{item.CRC & 0xFFFFFFFF:08x}" + ): + exact_match_count += 1 + else: + modified_count += 1 + additional_count = len(set(extracted_by_path) - scoped_paths) + multiset_verified = not (modified_count or missing_count or additional_count) + return { + "archive_sha256": archive_sha256, + "archive_size_bytes": archive_stat.st_size, + "archive_entry_count": len(all_entries), + "archive_uncompressed_size_bytes": sum(item.file_size for item in all_entries), + "archive_source_scope_entry_count": len(scoped_entries), + "archive_source_scope_size_bytes": sum(item.file_size for item, _ in scoped_entries), + "archive_original_entry_exact_match_count": exact_match_count, + "archive_original_entry_modified_count": modified_count, + "archive_original_entry_missing_count": missing_count, + "extracted_additional_file_count": additional_count, + "archive_integrity_verified": True, + "archive_extracted_entry_multiset_verified": multiset_verified, + } + + +def _source_scope_relative_path(item: zipfile.ZipInfo) -> str | None: + name = item.filename + if not item.flag_bits & 0x800: + try: + name = name.encode("cp437").decode("gbk") + except UnicodeError: + pass + marker = "/01数据样例/" + if marker not in name: + return None + return name.split(marker, 1)[1] + + +def _normalise_crs(value: Any) -> str: + text = str(value or "") + if text.startswith("COMPD_CS[") and 'AUTHORITY["EPSG","4523"]' in text: + return "EPSG:4523+EPSG:5737" + return text + + +def _source_group_for_profile(profile: Mapping[str, Any], source_root: Path) -> str: + source_path = Path(str(profile.get("source_path") or "")) + try: + relative = source_path.resolve().relative_to(source_root.resolve()).as_posix() + except ValueError as exc: + raise ChongqingRealSourceAdmissionError("asset profile escaped the source root") from exc + ordinal = relative.split("/", 1)[0][:2] + spec = GROUP_BY_ORDINAL.get(ordinal) + if spec is None: + raise ChongqingRealSourceAdmissionError("asset profile has an unknown source group") + return str(spec["source_group_id"]) + + +def _asset_profile( + raw: Mapping[str, Any], + *, + source_root: Path, +) -> dict[str, Any]: + asset_id = str(raw.get("asset_id") or "") + source_group_id = _source_group_for_profile(raw, source_root) + group_spec = GROUP_BY_ID[source_group_id] + metric_names = ( + "feature_count", + "row_count", + "column_count", + "sheet_count", + "width", + "height", + "band_count", + "pixel_count", + "layer_count", + "nonempty_layer_count", + ) + metrics = {name: int(raw[name]) for name in metric_names if raw.get(name) is not None} + crs = _normalise_crs(raw.get("crs")) + spatial: dict[str, Any] | None = None + if raw.get("geometry_type") is not None or crs or raw.get("bounds") is not None: + spatial = { + "geometry_type": str(raw.get("geometry_type") or ""), + "crs": crs, + "bounds": ( + [float(item) for item in raw["bounds"]] if raw.get("bounds") is not None else None + ), + "dtype": [str(item) for item in raw.get("dtype", [])], + "nodata": raw.get("nodata"), + } + fields = raw.get("fields") or raw.get("columns") or [] + stable = { + "asset_id": asset_id, + "source_group_id": source_group_id, + "source_ref": f"source://{SOURCE_ID}/assets/{asset_id}", + "asset_kind": str(raw.get("asset_kind") or ""), + "data_classification": group_spec["data_classification"], + "admission_status": STATUS, + "record_metrics": metrics, + "spatial_profile": spatial, + "schema_fields": [str(item) for item in fields], + "roles": sorted(item for item in str(raw.get("uwm_roles") or "").split(";") if item), + "metadata_profiled": True, + "source_content_in_evidence": False, + } + return {**stable, "profile_sha256": canonical_json_fingerprint(stable)} + + +def _asset_profiles(source_root: Path, source_zip: Path, captured_at: str) -> list[dict[str, Any]]: + report = scan_local_planning_zip_assets( + source_root=source_root, + source_zip=source_zip, + created_at=captured_at, + ) + raw_profiles = [ + *report.get("vector_profiles", []), + *report.get("tabular_profiles", []), + *report.get("raster_profiles", []), + ] + profiles = [_asset_profile(raw, source_root=source_root) for raw in raw_profiles] + return sorted(profiles, key=lambda item: str(item["asset_id"])) + + +def _research_audit_binding(path: Path = RESEARCH_INVENTORY_PATH) -> dict[str, Any]: + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + return { + "path": str(path.relative_to(REPO_ROOT)), + "file_sha256": _file_sha256(path), + "row_count": len(rows), + "asset_ids_sha256": canonical_json_fingerprint( + sorted(str(row.get("asset_id") or "") for row in rows) + ), + "authority": "research_inventory_only", + "admission_authority": False, + } + + +def build_evidence( + *, + source_root: Path, + source_zip: Path, + captured_at: datetime | None = None, + research_inventory_path: Path = RESEARCH_INVENTORY_PATH, +) -> dict[str, Any]: + timestamp = (captured_at or datetime.now(UTC)).astimezone(UTC) + captured_text = timestamp.isoformat().replace("+00:00", "Z") + payload_records = _scan_payload_files(source_root) + groups = _build_source_groups(payload_records) + archive = _archive_binding(source_zip, payload_records) + profiles = _asset_profiles(source_root, source_zip, captured_text) + blockers = sorted( + [ + *SOURCE_BINDING_BLOCKERS, + *(blocker for group in groups for blocker in group["blockers"]), + ] + ) + stable = { + "schema": EVIDENCE_SCHEMA, + "status": STATUS, + "captured_at": captured_text, + "source_binding": { + "source_id": SOURCE_ID, + **archive, + "extracted_file_count": len(payload_records), + "extracted_size_bytes": sum(int(item["size_bytes"]) for item in payload_records), + "extracted_payload_sha256": _payload_fingerprint(payload_records), + "logical_group_inventory_sha256": canonical_json_fingerprint(groups), + "source_payload_in_repository": False, + "absolute_source_paths_in_evidence": False, + }, + "research_audit_binding": _research_audit_binding(research_inventory_path), + "source_groups": groups, + "asset_profiles": profiles, + "admission_blockers": blockers, + "admission_policy": dict(ADMISSION_POLICY), + "claims": { + "archive_integrity_verified": archive["archive_integrity_verified"], + "archive_extracted_entry_multiset_verified": archive[ + "archive_extracted_entry_multiset_verified" + ], + "full_source_metadata_profiled": True, + "source_governance_approved": False, + "source_content_admitted": False, + "source_publication_authorized": False, + "scheduler_submission_authorized": False, + "provider_mutation_authorized": False, + "production_ingestion_verified": False, + "production_ready": False, + }, + } + return {**stable, "evidence_sha256": canonical_json_fingerprint(stable)} + + +def _sensitive_paths(value: Any, prefix: str = "") -> list[str]: + findings: list[str] = [] + if isinstance(value, Mapping): + for key, item in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if SENSITIVE_KEY_PATTERN.search(str(key)): + findings.append(path) + findings.extend(_sensitive_paths(item, path)) + elif isinstance(value, list): + for index, item in enumerate(value): + findings.extend(_sensitive_paths(item, f"{prefix}[{index}]")) + return findings + + +def _asset_baseline_errors(profile: Mapping[str, Any]) -> list[str]: + asset_id = str(profile.get("asset_id") or "") + expected = EXPECTED_ASSET_BASELINES.get(asset_id) + if expected is None: + return [f"M3-28 asset profile is unknown: {asset_id}"] + errors: list[str] = [] + for key in ("source_group_id", "asset_kind"): + if profile.get(key) != expected[key]: + errors.append(f"M3-28 asset {key} does not match: {asset_id}") + metrics = profile.get("record_metrics") + if not isinstance(metrics, Mapping): + errors.append(f"M3-28 asset record metrics are invalid: {asset_id}") + else: + for key, value in expected["record_metrics"].items(): + if metrics.get(key) != value: + errors.append(f"M3-28 asset metric does not match: {asset_id}.{key}") + spatial = profile.get("spatial_profile") + for key in ("geometry_type", "crs"): + if key not in expected: + continue + if not isinstance(spatial, Mapping) or spatial.get(key) != expected[key]: + errors.append(f"M3-28 asset spatial profile does not match: {asset_id}.{key}") + return errors + + +def validate_evidence(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if set(evidence) != EVIDENCE_INVENTORY: + errors.append("M3-28 evidence inventory does not match") + stable = {key: value for key, value in evidence.items() if key != "evidence_sha256"} + if evidence.get("evidence_sha256") != canonical_json_fingerprint(stable): + errors.append("M3-28 evidence fingerprint does not match") + if evidence.get("schema") != EVIDENCE_SCHEMA or evidence.get("status") != STATUS: + errors.append("M3-28 evidence schema or status does not match") + try: + _parse_time(evidence.get("captured_at")) + except ChongqingRealSourceAdmissionError as exc: + errors.append(str(exc)) + + source = evidence.get("source_binding") + if not isinstance(source, Mapping): + errors.append("M3-28 source binding is invalid") + source = {} + elif set(source) != SOURCE_BINDING_INVENTORY: + errors.append("M3-28 source binding inventory does not match") + expected_source_values = { + "source_id": SOURCE_ID, + "archive_sha256": EXPECTED_ARCHIVE_SHA256, + "archive_size_bytes": EXPECTED_ARCHIVE_SIZE_BYTES, + "archive_entry_count": EXPECTED_ARCHIVE_ENTRY_COUNT, + "archive_uncompressed_size_bytes": EXPECTED_ARCHIVE_UNCOMPRESSED_SIZE_BYTES, + "archive_source_scope_entry_count": EXPECTED_ARCHIVE_SCOPE_ENTRY_COUNT, + "archive_source_scope_size_bytes": EXPECTED_ARCHIVE_SCOPE_SIZE_BYTES, + "archive_original_entry_exact_match_count": EXPECTED_ARCHIVE_EXACT_MATCH_COUNT, + "archive_original_entry_modified_count": EXPECTED_ARCHIVE_MODIFIED_COUNT, + "archive_original_entry_missing_count": EXPECTED_ARCHIVE_MISSING_COUNT, + "extracted_additional_file_count": EXPECTED_EXTRACTED_ADDITIONAL_COUNT, + "archive_integrity_verified": True, + "archive_extracted_entry_multiset_verified": False, + "extracted_file_count": EXPECTED_EXTRACTED_FILE_COUNT, + "extracted_size_bytes": EXPECTED_EXTRACTED_SIZE_BYTES, + "source_payload_in_repository": False, + "absolute_source_paths_in_evidence": False, + } + for key, value in expected_source_values.items(): + if source.get(key) != value: + errors.append(f"M3-28 source binding does not match: {key}") + if ( + EXPECTED_EXTRACTED_PAYLOAD_SHA256 + and source.get("extracted_payload_sha256") != EXPECTED_EXTRACTED_PAYLOAD_SHA256 + ): + errors.append("M3-28 extracted payload fingerprint does not match") + for key in ( + "archive_sha256", + "extracted_payload_sha256", + "logical_group_inventory_sha256", + ): + if not SHA256_PATTERN.fullmatch(str(source.get(key) or "")): + errors.append(f"M3-28 source fingerprint is invalid: {key}") + + research = evidence.get("research_audit_binding") + if not isinstance(research, Mapping): + errors.append("M3-28 research audit binding is invalid") + research = {} + if ( + set(research) != RESEARCH_AUDIT_INVENTORY + or research.get("path") != str(RESEARCH_INVENTORY_PATH.relative_to(REPO_ROOT)) + or research.get("file_sha256") != EXPECTED_RESEARCH_INVENTORY_SHA256 + or research.get("row_count") != EXPECTED_RESEARCH_INVENTORY_ROWS + or research.get("asset_ids_sha256") != EXPECTED_RESEARCH_ASSET_IDS_SHA256 + or research.get("authority") != "research_inventory_only" + or research.get("admission_authority") is not False + ): + errors.append("M3-28 research audit binding does not match") + + if evidence.get("admission_policy") != ADMISSION_POLICY: + errors.append("M3-28 admission policy does not match") + + groups_value = evidence.get("source_groups") + groups = groups_value if isinstance(groups_value, list) else [] + if not isinstance(groups_value, list): + errors.append("M3-28 source groups are not a list") + group_ids: list[str] = [] + derived_blockers: list[str] = [] + file_count = 0 + size_bytes = 0 + for group in groups: + if not isinstance(group, Mapping): + errors.append("M3-28 source group is not an object") + continue + group_id = str(group.get("source_group_id") or "") + group_ids.append(group_id) + if set(group) != SOURCE_GROUP_INVENTORY: + errors.append(f"M3-28 source group inventory does not match: {group_id}") + spec = GROUP_BY_ID.get(group_id) + if spec is None: + errors.append(f"M3-28 source group is unknown: {group_id}") + continue + if ( + group.get("ordinal") != spec["ordinal"] + or group.get("data_domains") != spec["data_domains"] + or group.get("data_classification") != spec["data_classification"] + or group.get("blockers") != _source_group_blockers(spec) + ): + errors.append(f"M3-28 source group contract does not match: {group_id}") + for key, expected in ( + ("license_status", "unverified_restricted"), + ("admission_status", STATUS), + ("metadata_profiled", True), + ("content_admitted", False), + ): + if group.get(key) != expected: + errors.append(f"M3-28 source group claim does not match: {group_id}.{key}") + if not SHA256_PATTERN.fullmatch(str(group.get("content_manifest_sha256") or "")): + errors.append(f"M3-28 source group fingerprint is invalid: {group_id}") + extensions = group.get("extension_counts") + if not isinstance(extensions, Mapping) or sum( + int(value) for value in extensions.values() + ) != group.get("file_count"): + errors.append(f"M3-28 source group extension counts do not match: {group_id}") + file_count += int(group.get("file_count") or 0) + size_bytes += int(group.get("size_bytes") or 0) + derived_blockers.extend(str(item) for item in group.get("blockers", [])) + expected_group_ids = [str(spec["source_group_id"]) for spec in SOURCE_GROUP_SPECS] + if group_ids != expected_group_ids or len(group_ids) != len(set(group_ids)): + errors.append("M3-28 source group inventory is incomplete or reordered") + if file_count != EXPECTED_EXTRACTED_FILE_COUNT or size_bytes != EXPECTED_EXTRACTED_SIZE_BYTES: + errors.append("M3-28 source group physical totals do not match") + if source.get("logical_group_inventory_sha256") != canonical_json_fingerprint(groups): + errors.append("M3-28 logical group inventory fingerprint does not match") + + profiles_value = evidence.get("asset_profiles") + profiles = profiles_value if isinstance(profiles_value, list) else [] + if not isinstance(profiles_value, list): + errors.append("M3-28 asset profiles are not a list") + asset_ids: list[str] = [] + for profile in profiles: + if not isinstance(profile, Mapping): + errors.append("M3-28 asset profile is not an object") + continue + asset_id = str(profile.get("asset_id") or "") + asset_ids.append(asset_id) + if set(profile) != ASSET_PROFILE_INVENTORY: + errors.append(f"M3-28 asset profile inventory does not match: {asset_id}") + profile_stable = {key: value for key, value in profile.items() if key != "profile_sha256"} + if profile.get("profile_sha256") != canonical_json_fingerprint(profile_stable): + errors.append(f"M3-28 asset profile fingerprint does not match: {asset_id}") + if not SOURCE_REF_PATTERN.fullmatch(str(profile.get("source_ref") or "")): + errors.append(f"M3-28 asset source reference is invalid: {asset_id}") + group_spec = GROUP_BY_ID.get(str(profile.get("source_group_id") or "")) + if ( + group_spec is None + or profile.get("data_classification") != group_spec["data_classification"] + or profile.get("admission_status") != STATUS + or profile.get("metadata_profiled") is not True + or profile.get("source_content_in_evidence") is not False + ): + errors.append(f"M3-28 asset admission boundary does not match: {asset_id}") + errors.extend(_asset_baseline_errors(profile)) + expected_asset_ids = sorted(EXPECTED_ASSET_BASELINES) + if asset_ids != expected_asset_ids or len(asset_ids) != len(set(asset_ids)): + errors.append("M3-28 asset profile inventory is incomplete or reordered") + + blockers = evidence.get("admission_blockers") + expected_blockers = sorted([*SOURCE_BINDING_BLOCKERS, *derived_blockers]) + if blockers != expected_blockers or len(expected_blockers) != len(set(expected_blockers)): + errors.append("M3-28 admission blockers do not match source groups") + + claims = evidence.get("claims") + if not isinstance(claims, Mapping) or set(claims) != CLAIMS: + errors.append("M3-28 claims inventory does not match") + else: + expected_true = { + "archive_integrity_verified", + "full_source_metadata_profiled", + } + for claim in CLAIMS: + expected = claim in expected_true + if claims.get(claim) is not expected: + errors.append(f"M3-28 claim does not match: {claim}") + if _sensitive_paths(evidence): + errors.append("M3-28 evidence contains credential-bearing fields") + rendered = json.dumps(evidence, ensure_ascii=False, sort_keys=True) + for forbidden in ("/Users/", "Downloads/", ".tmp/twm_standard_1128"): + if forbidden in rendered: + errors.append("M3-28 evidence contains a local source path") + break + return sorted(set(errors)) + + +def build_validation_report( + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + try: + file_sha256 = _file_sha256(evidence_path) + evidence = _load_json_object(evidence_path) + errors = validate_evidence(evidence) + if EVIDENCE_FILE_SHA256 and file_sha256 != EVIDENCE_FILE_SHA256: + errors.append("M3-28 evidence file fingerprint does not match") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + evidence = {} + file_sha256 = None + errors = [f"M3-28 evidence is unreadable: {type(exc).__name__}"] + source = evidence.get("source_binding") + return { + "schema": VALIDATION_SCHEMA, + "status": "valid" if not errors else "invalid", + "errors": sorted(set(errors)), + "evidence_file_sha256": file_sha256, + "evidence_sha256": evidence.get("evidence_sha256"), + "source_status": evidence.get("status"), + "extracted_file_count": ( + source.get("extracted_file_count") if isinstance(source, Mapping) else None + ), + "source_group_count": len(evidence.get("source_groups", [])), + "asset_profile_count": len(evidence.get("asset_profiles", [])), + "admission_blocker_count": len(evidence.get("admission_blockers", [])), + "source_content_admitted": ( + evidence.get("claims", {}).get("source_content_admitted") + if isinstance(evidence.get("claims"), Mapping) + else None + ), + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + snapshot = subparsers.add_parser("snapshot") + snapshot.add_argument("--source-root", type=Path, required=True) + snapshot.add_argument("--source-zip", type=Path, required=True) + snapshot.add_argument("--output", type=Path, default=DEFAULT_EVIDENCE_PATH) + snapshot.add_argument( + "--research-inventory", + type=Path, + default=RESEARCH_INVENTORY_PATH, + ) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report(args.evidence) + exit_code = 0 if report["status"] == "valid" else 1 + else: + report = build_evidence( + source_root=args.source_root, + source_zip=args.source_zip, + research_inventory_path=args.research_inventory, + ) + errors = validate_evidence(report) + if errors: + raise ChongqingRealSourceAdmissionError("; ".join(errors)) + _write_json(args.output, report) + exit_code = 0 + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + return exit_code + except ( + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + zipfile.BadZipFile, + ChongqingRealSourceAdmissionError, + ) as exc: + print(f"Chongqing real-source admission: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/test_chongqing_real_source_admission.py b/data_agent/test_chongqing_real_source_admission.py new file mode 100644 index 00000000..8d8585d9 --- /dev/null +++ b/data_agent/test_chongqing_real_source_admission.py @@ -0,0 +1,194 @@ +import json +import zipfile +from copy import deepcopy +from datetime import UTC, datetime + +from data_agent import chongqing_real_source_admission as admission + +CAPTURED_AT = datetime(2026, 7, 31, 9, 30, tzinfo=UTC) + + +def _checked_evidence() -> dict: + return json.loads(admission.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _rehash(value: dict) -> None: + for profile in value.get("asset_profiles", []): + stable = {key: item for key, item in profile.items() if key != "profile_sha256"} + profile["profile_sha256"] = admission.canonical_json_fingerprint(stable) + stable = {key: item for key, item in value.items() if key != "evidence_sha256"} + value["evidence_sha256"] = admission.canonical_json_fingerprint(stable) + + +def test_checked_full_source_evidence_is_valid_and_blocked(): + evidence = _checked_evidence() + report = admission.build_validation_report() + + assert admission.validate_evidence(evidence) == [] + assert report["status"] == "valid" + assert report["errors"] == [] + assert report["extracted_file_count"] == 584 + assert report["source_group_count"] == 11 + assert report["asset_profile_count"] == 16 + assert evidence["status"] == admission.STATUS + assert evidence["claims"]["source_content_admitted"] is False + assert evidence["claims"]["scheduler_submission_authorized"] is False + assert evidence["claims"]["provider_mutation_authorized"] is False + assert evidence["claims"]["production_ready"] is False + + +def test_checked_evidence_binds_real_archive_and_all_extracted_bytes(): + evidence = _checked_evidence() + source = evidence["source_binding"] + + assert source["archive_sha256"] == admission.EXPECTED_ARCHIVE_SHA256 + assert source["archive_size_bytes"] == 468_462_251 + assert source["archive_entry_count"] == 533 + assert source["archive_uncompressed_size_bytes"] == 694_164_379 + assert source["archive_source_scope_entry_count"] == 532 + assert source["archive_source_scope_size_bytes"] == 694_147_946 + assert source["extracted_file_count"] == 584 + assert source["extracted_size_bytes"] == 700_610_744 + assert source["archive_original_entry_exact_match_count"] == 526 + assert source["archive_original_entry_modified_count"] == 6 + assert source["archive_original_entry_missing_count"] == 0 + assert source["extracted_additional_file_count"] == 52 + assert source["archive_integrity_verified"] is True + assert source["archive_extracted_entry_multiset_verified"] is False + assert source["source_payload_in_repository"] is False + assert source["absolute_source_paths_in_evidence"] is False + + +def test_source_groups_cover_physical_inventory_and_stay_unadmitted(): + evidence = _checked_evidence() + groups = evidence["source_groups"] + + assert [group["source_group_id"] for group in groups] == [ + spec["source_group_id"] for spec in admission.SOURCE_GROUP_SPECS + ] + assert sum(group["file_count"] for group in groups) == 584 + assert sum(group["size_bytes"] for group in groups) == 700_610_744 + assert all(group["metadata_profiled"] is True for group in groups) + assert all(group["content_admitted"] is False for group in groups) + assert all(group["license_status"] == "unverified_restricted" for group in groups) + assert all(group["blockers"] for group in groups) + assert len(evidence["admission_blockers"]) == len(set(evidence["admission_blockers"])) + + +def test_asset_profiles_correct_old_village_layer_scope_and_bind_baselines(): + evidence = _checked_evidence() + profiles = {profile["asset_id"]: profile for profile in evidence["asset_profiles"]} + village = profiles["fulu_village_planning_database_local"] + + assert set(profiles) == set(admission.EXPECTED_ASSET_BASELINES) + assert village["record_metrics"] == { + "feature_count": 8050, + "layer_count": 28, + "nonempty_layer_count": 20, + } + assert profiles["gaode_poi_2024"]["record_metrics"]["feature_count"] == 1_194_351 + assert profiles["chongqing_clcd_2020"]["record_metrics"]["pixel_count"] == 280_208_478 + assert profiles["chongqing_unicom_commuting_2023_local"]["data_classification"] == ( + "highly_restricted_aggregate_mobility" + ) + assert all(profile["source_content_in_evidence"] is False for profile in profiles.values()) + + +def test_evidence_is_path_free_and_contains_no_source_values(): + rendered = admission.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8") + evidence = json.loads(rendered) + + assert "/Users/" not in rendered + assert "Downloads/" not in rendered + assert ".tmp/twm_standard_1128" not in rendered + assert "geometry_values" not in rendered + assert "od_rows" not in rendered + assert "flow_rows" not in rendered + assert admission._sensitive_paths(evidence) == [] + assert all( + profile["source_ref"].startswith("source://chongqing-planning-institute-sample/assets/") + for profile in evidence["asset_profiles"] + ) + + +def test_outer_rehash_cannot_hide_admission_or_baseline_overclaim(): + evidence = deepcopy(_checked_evidence()) + evidence["claims"]["source_content_admitted"] = True + evidence["admission_policy"]["ci_requires_local_source_payload"] = True + evidence["source_binding"]["local_source_path"] = "/untrusted/source" + evidence["research_audit_binding"]["admission_authority"] = True + evidence["source_groups"][0]["content_admitted"] = True + profile = next( + item for item in evidence["asset_profiles"] if item["asset_id"] == "gaode_poi_2024" + ) + profile["record_metrics"]["feature_count"] = 1 + _rehash(evidence) + + errors = admission.validate_evidence(evidence) + + assert "M3-28 claim does not match: source_content_admitted" in errors + assert "M3-28 admission policy does not match" in errors + assert "M3-28 source binding inventory does not match" in errors + assert "M3-28 research audit binding does not match" in errors + assert any("source group claim does not match" in error for error in errors) + assert "M3-28 asset metric does not match: gaode_poi_2024.feature_count" in errors + + +def test_checked_file_fingerprint_rejects_rehashed_copy(tmp_path): + evidence = deepcopy(_checked_evidence()) + evidence["captured_at"] = "2026-07-31T10:00:00Z" + _rehash(evidence) + path = tmp_path / "evidence.json" + path.write_text(json.dumps(evidence, ensure_ascii=False), encoding="utf-8") + + report = admission.build_validation_report(path) + + assert report["status"] == "invalid" + assert "M3-28 evidence file fingerprint does not match" in report["errors"] + + +def test_archive_and_extracted_entry_multiset_is_verified(tmp_path): + root = tmp_path / "source" + group = root / "01-dem" + group.mkdir(parents=True) + payload = group / "tile.bin" + payload.write_bytes(b"real-source-content") + archive_path = tmp_path / "source.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.write(payload, "bundle/01数据样例/01-dem/tile.bin") + + records = admission._scan_payload_files(root) + binding = admission._archive_binding(archive_path, records) + + assert len(records) == 1 + assert binding["archive_entry_count"] == 1 + assert binding["archive_source_scope_entry_count"] == 1 + assert binding["archive_original_entry_exact_match_count"] == 1 + assert binding["archive_original_entry_modified_count"] == 0 + assert binding["extracted_additional_file_count"] == 0 + assert binding["archive_integrity_verified"] is True + assert binding["archive_extracted_entry_multiset_verified"] is True + + +def test_source_scan_rejects_symbolic_links(tmp_path): + root = tmp_path / "source" + group = root / "01-dem" + group.mkdir(parents=True) + target = tmp_path / "outside.bin" + target.write_bytes(b"outside") + (group / "linked.bin").symlink_to(target) + + try: + admission._scan_payload_files(root) + except admission.ChongqingRealSourceAdmissionError as exc: + assert "symbolic link" in str(exc) + else: + raise AssertionError("symbolic link was accepted") + + +def test_wrapper_is_strict_and_invokes_source_admission(): + wrapper = admission.REPO_ROOT / "scripts/chongqing-real-source-admission.sh" + text = wrapper.read_text(encoding="utf-8") + + assert "set -euo pipefail" in text + assert "chongqing_real_source_admission" in text diff --git a/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md b/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md new file mode 100644 index 00000000..f2af3501 --- /dev/null +++ b/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md @@ -0,0 +1,58 @@ +# ADR-074: Chongqing real-source admission manifest + +**Status**: Accepted + +**Date**: 2026-07-31 + +## Context + +AR-2 needs representative real sources before it can define ingestion, quality and lakehouse contracts. The available Chongqing planning-institute sample spans elevation, roads, remote-sensing land cover, buildings, cultural-planning data, population, POI/AOI, commuting and search activity. Earlier work profiled 16 useful assets, but that research inventory did not bind the complete source payload and had no authority to admit content. + +The original ZIP and the current extracted working set are not identical. All 532 archive entries in the `01数据样例` scope are present, but only 526 match by size and CRC, 6 differ and 52 files exist only in the extracted set. The additional material includes expanded FileGDB content and generated sidecars. Without derivation provenance, treating the extracted set as a byte-exact extraction would create a false source claim. + +## Decision + +Adopt the checked M3-28 manifest as a path-free, content-addressed and metadata-only admission contract for the full Chongqing source set. + +The manifest binds: + +- the 468,462,251-byte source archive with SHA-256 `2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca`, 533 files and 694,164,379 uncompressed bytes; +- the 532-entry archive source scope with 694,147,946 bytes; +- the 584-file, 700,610,744-byte extracted working set with payload fingerprint `e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6`; +- 11 complete physical source groups and 16 metadata asset profiles; +- the earlier 16-row research inventory as `research_inventory_only`, never as admission authority. + +The checked evidence contains aggregate physical inventories, schema field names, counts, CRS, bounds, roles and content fingerprints. It contains no absolute source path, source file, geometry, row value, credential or record-level payload. + +The validator freezes the archive/extracted comparison at 526 exact matches, 6 modified entries, no missing archive entry and 52 extracted-only files. It therefore requires `archive_integrity_verified=true` but `archive_extracted_entry_multiset_verified=false`, and retains `source_binding:extraction_derivation_provenance_missing` as an admission blocker. + +Every source group also remains blocked on owner, license, retention and access decisions, plus domain-specific privacy, commercial terms, attribution, vintage, lineage or sensitivity review where applicable. The manifest reports 57 unique blockers. + +## Authority boundary + +Metadata profiling is allowed; content admission is not. The manifest does not create a `ResourceVersion`, `PolicyDecision`, `Approval`, `PlatformRun`, scheduler command, provider mutation, landing object, lakehouse table or data product. + +The local ZIP remains the observed archive payload and the extracted directory remains a working set. Neither local path is a production source authority. The checked JSON is evidence about those bytes, not a copy of them and not authority for owner, license, retention, access, privacy or publication decisions. + +CI validates only the checked evidence and never requires the local source paths. Any future admission must resolve every blocker, preserve derivation provenance and execute a fresh protected ingestion. M3-24/M3-25 retained material cannot be promoted as a substitute. + +## Consequences + +**Positive**: AR-2 now has a reproducible real-source baseline covering the full available Chongqing corpus instead of a synthetic fixture or one 20-feature slice. + +**Positive**: archive integrity, extracted working-set identity and asset metadata are independently bound, so later ingestion can detect source drift without committing restricted data. + +**Positive**: the earlier village-planning scope is corrected to 28 Shapefile layers, 20 non-empty layers and 8,050 features; the prior count of 31 described all Shapefiles under the root, not the village subset. + +**Negative**: M3-28 intentionally admits no content and leaves 57 blockers unresolved. AR-2 is `in_progress`, not verified. + +**Negative**: the extracted working set cannot become an admitted landing source until its six modified and 52 additional files have documented derivation provenance. + +## Verification + +```bash +./scripts/chongqing-real-source-admission.sh validate +python -m pytest data_agent/test_chongqing_real_source_admission.py -q +``` + +The checked evidence fingerprint is `a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1`; its file SHA-256 is `9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83`. diff --git a/docs/evidence/chongqing-real-source-admission-2026-07-31.json b/docs/evidence/chongqing-real-source-admission-2026-07-31.json new file mode 100644 index 00000000..e1115b9b --- /dev/null +++ b/docs/evidence/chongqing-real-source-admission-2026-07-31.json @@ -0,0 +1,1119 @@ +{ + "admission_blockers": [ + "source_binding:extraction_derivation_provenance_missing", + "source_group:baidu-aoi-2024:access_policy_missing", + "source_group:baidu-aoi-2024:contact_field_privacy_review_missing", + "source_group:baidu-aoi-2024:license_terms_unverified", + "source_group:baidu-aoi-2024:owner_reference_missing", + "source_group:baidu-aoi-2024:retention_policy_missing", + "source_group:baidu-search-index-2023:access_policy_missing", + "source_group:baidu-search-index-2023:commercial_terms_review_missing", + "source_group:baidu-search-index-2023:license_terms_unverified", + "source_group:baidu-search-index-2023:owner_reference_missing", + "source_group:baidu-search-index-2023:retention_policy_missing", + "source_group:bishan-planning-materials:access_policy_missing", + "source_group:bishan-planning-materials:license_terms_unverified", + "source_group:bishan-planning-materials:owner_reference_missing", + "source_group:bishan-planning-materials:planning_sensitivity_review_missing", + "source_group:bishan-planning-materials:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:access_policy_missing", + "source_group:chongqing-central-buildings-2021:license_terms_unverified", + "source_group:chongqing-central-buildings-2021:owner_reference_missing", + "source_group:chongqing-central-buildings-2021:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:source_vintage_unverified", + "source_group:chongqing-clcd-2020:access_policy_missing", + "source_group:chongqing-clcd-2020:classification_lineage_unverified", + "source_group:chongqing-clcd-2020:license_terms_unverified", + "source_group:chongqing-clcd-2020:owner_reference_missing", + "source_group:chongqing-clcd-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:access_policy_missing", + "source_group:chongqing-dem-2020:license_terms_unverified", + "source_group:chongqing-dem-2020:owner_reference_missing", + "source_group:chongqing-dem-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:source_vintage_unverified", + "source_group:chongqing-district-population-2021:access_policy_missing", + "source_group:chongqing-district-population-2021:license_terms_unverified", + "source_group:chongqing-district-population-2021:owner_reference_missing", + "source_group:chongqing-district-population-2021:retention_policy_missing", + "source_group:chongqing-district-population-2021:statistics_vintage_review_missing", + "source_group:chongqing-historic-districts:access_policy_missing", + "source_group:chongqing-historic-districts:license_terms_unverified", + "source_group:chongqing-historic-districts:owner_reference_missing", + "source_group:chongqing-historic-districts:retention_policy_missing", + "source_group:chongqing-historic-districts:source_vintage_unverified", + "source_group:chongqing-osm-roads-2021:access_policy_missing", + "source_group:chongqing-osm-roads-2021:license_terms_unverified", + "source_group:chongqing-osm-roads-2021:odbl_attribution_policy_missing", + "source_group:chongqing-osm-roads-2021:owner_reference_missing", + "source_group:chongqing-osm-roads-2021:retention_policy_missing", + "source_group:gaode-poi-2024:access_policy_missing", + "source_group:gaode-poi-2024:contact_field_privacy_review_missing", + "source_group:gaode-poi-2024:license_terms_unverified", + "source_group:gaode-poi-2024:owner_reference_missing", + "source_group:gaode-poi-2024:retention_policy_missing", + "source_group:unicom-commuting-2023:access_policy_missing", + "source_group:unicom-commuting-2023:grid_geometry_dictionary_missing", + "source_group:unicom-commuting-2023:license_terms_unverified", + "source_group:unicom-commuting-2023:owner_reference_missing", + "source_group:unicom-commuting-2023:privacy_impact_assessment_missing", + "source_group:unicom-commuting-2023:retention_policy_missing" + ], + "admission_policy": { + "ci_requires_local_source_payload": false, + "content_admission_requires_owner_license_retention_and_access_approval": true, + "fresh_protected_ingestion_required_after_admission": true, + "local_profile_is_not_production_admission": true, + "metadata_profiling_allowed": true, + "restricted_fields_require_privacy_review": true, + "source_payload_copy_to_repository_forbidden": true + }, + "asset_profiles": [ + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "baidu_aoi_2024", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_location", + "metadata_profiled": true, + "profile_sha256": "2b6ce471d518ff7205eb9d5690e40cf7de39de1328260276b39673afad4ad9bc", + "record_metrics": { + "feature_count": 26292 + }, + "roles": [ + "baseline", + "planner_targeting", + "service_accessibility", + "urban_form" + ], + "schema_fields": [ + "uid", + "名称", + "地址", + "省份", + "城市", + "区县", + "街镇乡", + "类型", + "第一分类", + "第二分类", + "评分", + "更新时间", + "评论数", + "开业时间", + "人均价格_元", + "街道id", + "电话", + "创建时间", + "其他", + "高德分类", + "经度wgs84", + "纬度wgs84", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "baidu-aoi-2024", + "source_ref": "source://chongqing-planning-institute-sample/assets/baidu_aoi_2024", + "spatial_profile": { + "bounds": [ + 105.31552411900009, + 28.18493382400004, + 110.16574626100004, + 32.16731932000005 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "baidu_search_index_2023_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_activity", + "metadata_profiled": true, + "profile_sha256": "b6bc2eeaa4e0b6626551a9e7732632ee5983ae5572709d109d4996665dbf3202", + "record_metrics": { + "feature_count": 325 + }, + "roles": [ + "mmfe_alignment", + "mobility_activity", + "planner_targeting", + "simulator_context", + "urban_activity_proxy" + ], + "schema_fields": [ + "id", + "ODJSMC", + "DDJSMC", + "PCSSCS", + "YDSSCS", + "SSZS", + "Shape_Length" + ], + "source_content_in_evidence": false, + "source_group_id": "baidu-search-index-2023", + "source_ref": "source://chongqing-planning-institute-sample/assets/baidu_search_index_2023_local", + "spatial_profile": { + "bounds": [ + 102.66382204100006, + 27.822151572568785, + 111.13525443382999, + 33.09055455500004 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "MultiLineString", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_admin_boundary_cjdcq_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "0c7607470d2b0049e23b86639eba4e5109a0909e7f40bbeb357639ccb6e2ed9c", + "record_metrics": { + "feature_count": 1488 + }, + "roles": [ + "administrative_units", + "baseline", + "land_use_context", + "planning_constraints" + ], + "schema_fields": [ + "BSM", + "YSDM", + "ZLDWDM", + "ZLDWMC", + "DCMJ", + "JSMJ", + "MSSM", + "HDMC", + "BZ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_admin_boundary_cjdcq_local", + "spatial_profile": { + "bounds": [ + 35601015.4985, + 3241091.9637, + 35632312.22315, + 3308361.5723 + ], + "crs": "EPSG:4523+EPSG:5737", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_admin_boundary_xzq_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "384675e6d9888d701b9ae88875fcc11488f45ee7778bdf729a31cba8e8270d70", + "record_metrics": { + "feature_count": 15 + }, + "roles": [ + "administrative_units", + "baseline", + "land_use_context", + "planning_constraints" + ], + "schema_fields": [ + "BSM", + "YSDM", + "XZQDM", + "XZQMC", + "DCMJ", + "JSMJ", + "MSSM", + "HDMC", + "BZ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_admin_boundary_xzq_local", + "spatial_profile": { + "bounds": [ + 35601015.4985, + 3241091.9637, + 35632312.22315, + 3308361.5723 + ], + "crs": "EPSG:4523+EPSG:5737", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_land_development_ledger_2019_local", + "asset_kind": "workbook_collection", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "b62a42e2cf0c1a3a269ecaf53edb583ce5ce066ffd95e1e6cc28ca9357435cb7", + "record_metrics": { + "row_count": 1438, + "sheet_count": 4 + }, + "roles": [ + "land_development_pressure", + "planner_constraints", + "planning_context", + "simulator_context" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_land_development_ledger_2019_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_land_use_dltb_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "dd0f4eab16f69cc59fe320eeeff3aed747ed8de35a948a5cb87ee649ae3813c9", + "record_metrics": { + "feature_count": 101657 + }, + "roles": [ + "baseline", + "land_use_context", + "planning_constraints", + "simulator_context", + "urban_form" + ], + "schema_fields": [ + "BSM", + "YSDM", + "DLBM", + "DLMC", + "QSDWDM", + "QSDWMC", + "ZLDWDM", + "ZLDWMC", + "TBMJ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_land_use_dltb_local", + "spatial_profile": { + "bounds": [ + 106.04001576200005, + 29.282454706000067, + 106.36873229600008, + 29.887474954000083 + ], + "crs": "EPSG:4610", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_central_buildings_2021", + "asset_kind": "vector", + "data_classification": "restricted_local_built_environment", + "metadata_profiled": true, + "profile_sha256": "2e4997ea793c5a841a03a4da56e6dffc2962f196cbbaa1f8a3e8b5bc091dbce7", + "record_metrics": { + "feature_count": 107452 + }, + "roles": [ + "baseline", + "renderer", + "urban_form" + ], + "schema_fields": [ + "Id", + "Floor" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-central-buildings-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_central_buildings_2021", + "spatial_profile": { + "bounds": [ + 106.20951745600001, + 29.212573738600042, + 106.821612684, + 29.831229147900103 + ], + "crs": "EPSG:4326", + "dtype": [], + "geometry_type": "Polygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_clcd_2020", + "asset_kind": "raster", + "data_classification": "restricted_local_remote_sensing", + "metadata_profiled": true, + "profile_sha256": "653f7c175ff192b3fc4135eee60aa0754e389fe0de7b8301718f66148bf63571", + "record_metrics": { + "band_count": 1, + "height": 15082, + "pixel_count": 280208478, + "width": 18579 + }, + "roles": [ + "baseline", + "remote_sensing_state", + "urban_form" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-clcd-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_clcd_2020", + "spatial_profile": { + "bounds": [ + 105.21303750199797, + 28.140836217657494, + 110.21997740109495, + 32.20535355218468 + ], + "crs": "EPSG:4326", + "dtype": [ + "uint8" + ], + "geometry_type": "", + "nodata": 15.0 + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_dem_80m", + "asset_kind": "raster", + "data_classification": "restricted_local_baseline", + "metadata_profiled": true, + "profile_sha256": "a145f700de59f65a895bcace518250648db4ee130c147e7ff262fb529976893e", + "record_metrics": { + "band_count": 1, + "height": 1454, + "pixel_count": 2567764, + "width": 1766 + }, + "roles": [ + "heat_exposure", + "renderer" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-dem-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_dem_80m", + "spatial_profile": { + "bounds": [ + 105.28983805000004, + 28.165268865555987, + 110.19539360948005, + 32.204157757675986 + ], + "crs": "EPSG:4490", + "dtype": [ + "int16" + ], + "geometry_type": "", + "nodata": 32767.0 + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_district_population_stats_2021_local", + "asset_kind": "workbook", + "data_classification": "restricted_aggregate_population", + "metadata_profiled": true, + "profile_sha256": "d16f10876967f0c31d2dba850bda33fe82c0df2dde71945a7d239504a6a83840", + "record_metrics": { + "row_count": 41, + "sheet_count": 1 + }, + "roles": [ + "equity_evaluation", + "population_vulnerability" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-district-population-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_district_population_stats_2021_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_historic_districts_local", + "asset_kind": "vector", + "data_classification": "restricted_local_cultural_planning", + "metadata_profiled": true, + "profile_sha256": "d51869b52b11220da25690d7e890c14da7c0188b2ef381716daa07b58fceb385", + "record_metrics": { + "feature_count": 20 + }, + "roles": [ + "cultural_heritage", + "livability_context", + "planner_targeting", + "renderer", + "service_accessibility", + "urban_form" + ], + "schema_fields": [ + "OBJECTID", + "Jqmc", + "Fwlx", + "fwmc", + "Xzqmc", + "Fwkzyq", + "Fwmj", + "Bhbkydwwsl", + "Bhbkydwwzj", + "Bhlsjzsl", + "Bhlsjzmc", + "Bhlsjzzjzm", + "Bhlshjyssl", + "Bhlshjysmc", + "Bhlsjxsl", + "Bhlsjxzcd", + "Bhlsjxmcjc", + "Bhfwzwhycm", + "Bhqtfwzycs", + "Bhqtfwzycm", + "Bz", + "Jj", + "Bsm", + "Bhctfmjzsl", + "Bhbkydwwmc", + "Bhctfmjzzj", + "Bhgsmmsl", + "Bhgsmmmc", + "Bhfwzwhy_1", + "Tymj", + "Tycd", + "Shape_Leng", + "Shape_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-historic-districts", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_historic_districts_local", + "spatial_profile": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300008, + 29.877271985000025 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "Polygon Z", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_osm_roads_2021", + "asset_kind": "vector", + "data_classification": "restricted_local_open_data_copy", + "metadata_profiled": true, + "profile_sha256": "ff1129bc6b54ae32891e8dbeea0262942155ea4a08e89393720fd1cbd799fbe6", + "record_metrics": { + "feature_count": 50366 + }, + "roles": [ + "baseline", + "mobility_activity", + "mobility_graph", + "renderer" + ], + "schema_fields": [ + "osm_id", + "code", + "fclass", + "name", + "ref", + "oneway", + "maxspeed", + "layer", + "bridge", + "tunnel" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-osm-roads-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_osm_roads_2021", + "spatial_profile": { + "bounds": [ + 105.30804952870155, + 28.163572226020626, + 110.1732225690069, + 32.1562025 + ], + "crs": "EPSG:4326", + "dtype": [], + "geometry_type": "LineString", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_unicom_commuting_2023_local", + "asset_kind": "table", + "data_classification": "highly_restricted_aggregate_mobility", + "metadata_profiled": true, + "profile_sha256": "d7d0661e3e97356ca74d2e601d76c4b30184691f0266e8a27c8284eaade7008b", + "record_metrics": { + "column_count": 7, + "row_count": 2120 + }, + "roles": [ + "commuting_od", + "mobility_activity", + "planner_targeting", + "population_vulnerability", + "simulator_context" + ], + "schema_fields": [ + "居住格网", + "工作格网", + "职住格网是否重合", + "性别", + "年龄", + "扩样前人口", + "扩样后人口" + ], + "source_content_in_evidence": false, + "source_group_id": "unicom-commuting-2023", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_unicom_commuting_2023_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "clcd_classification_dictionary_local", + "asset_kind": "workbook", + "data_classification": "restricted_local_remote_sensing", + "metadata_profiled": true, + "profile_sha256": "5e2dead48038cf37d37ddcf69ee28be06d5bc636c5c390d9602a0ee2c60bc21d", + "record_metrics": { + "row_count": 10, + "sheet_count": 1 + }, + "roles": [ + "baseline_context", + "remote_sensing_state", + "renderer" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-clcd-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/clcd_classification_dictionary_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "fulu_village_planning_database_local", + "asset_kind": "vector_collection", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "a7046403283e32c6972382b882940407f3a6205a4d6f63561863d8e1cb24d743", + "record_metrics": { + "feature_count": 8050, + "layer_count": 28, + "nonempty_layer_count": 20 + }, + "roles": [ + "land_use_context", + "planning_constraints", + "simulator_context", + "village_livability_context" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/fulu_village_planning_database_local", + "spatial_profile": { + "bounds": null, + "crs": "mixed_CGCS2000_GK_zone_35_EPSG4523", + "dtype": [], + "geometry_type": "mixed", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "gaode_poi_2024", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_location", + "metadata_profiled": true, + "profile_sha256": "fc8e70f04bfab1c9305a1edff49d2cc39d9c1eee4f71d5b8bfe99780ed57dd5c", + "record_metrics": { + "feature_count": 1194351 + }, + "roles": [ + "baseline", + "planner_targeting", + "service_accessibility" + ], + "schema_fields": [ + "ID", + "名称", + "地址", + "电话", + "类型", + "区域ID", + "经度wgs84", + "纬度wgs84", + "百度经度", + "百度纬度", + "更新时间" + ], + "source_content_in_evidence": false, + "source_group_id": "gaode-poi-2024", + "source_ref": "source://chongqing-planning-institute-sample/assets/gaode_poi_2024", + "spatial_profile": { + "bounds": [ + 105.28943500000003, + 28.164064000000053, + 110.18358100000006, + 32.190248000000054 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "Point", + "nodata": null + } + } + ], + "captured_at": "2026-07-31T07:54:06.279487Z", + "claims": { + "archive_extracted_entry_multiset_verified": false, + "archive_integrity_verified": true, + "full_source_metadata_profiled": true, + "production_ingestion_verified": false, + "production_ready": false, + "provider_mutation_authorized": false, + "scheduler_submission_authorized": false, + "source_content_admitted": false, + "source_governance_approved": false, + "source_publication_authorized": false + }, + "evidence_sha256": "a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1", + "research_audit_binding": { + "admission_authority": false, + "asset_ids_sha256": "c7cb765c2b653dbf5619e2fe1cf027d108a5951b752b6dd4bfd60b4c00d91947", + "authority": "research_inventory_only", + "file_sha256": "69b4167955f950041988dd174b75ea5376af146ef3e52f815aa57715fd24f70d", + "path": "data/uwm_public_proxy/chongqing_central/local_planning_zip_audit_2026_07_05/uwm_local_planning_zip_inventory.csv", + "row_count": 16 + }, + "schema": "gda.chongqing_real_source_admission.v1", + "source_binding": { + "absolute_source_paths_in_evidence": false, + "archive_entry_count": 533, + "archive_extracted_entry_multiset_verified": false, + "archive_integrity_verified": true, + "archive_original_entry_exact_match_count": 526, + "archive_original_entry_missing_count": 0, + "archive_original_entry_modified_count": 6, + "archive_sha256": "2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca", + "archive_size_bytes": 468462251, + "archive_source_scope_entry_count": 532, + "archive_source_scope_size_bytes": 694147946, + "archive_uncompressed_size_bytes": 694164379, + "extracted_additional_file_count": 52, + "extracted_file_count": 584, + "extracted_payload_sha256": "e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6", + "extracted_size_bytes": 700610744, + "logical_group_inventory_sha256": "5aa5f1aebd135956c550f392deb5965e7c088d4cb09070b60b4db45e5ce9503b", + "source_id": "chongqing-planning-institute-sample", + "source_payload_in_repository": false + }, + "source_groups": [ + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-dem-2020:owner_reference_missing", + "source_group:chongqing-dem-2020:license_terms_unverified", + "source_group:chongqing-dem-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:access_policy_missing", + "source_group:chongqing-dem-2020:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "9796f99f214d6ae1e9cef659ffe341465ec48a93a25bba1b966c4ba3052630b0", + "data_classification": "restricted_local_baseline", + "data_domains": [ + "elevation", + "terrain", + "raster" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "ovr": 1, + "tfw": 1, + "tif": 1, + "xml": 2 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "01", + "size_bytes": 2360205, + "source_group_id": "chongqing-dem-2020" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-osm-roads-2021:owner_reference_missing", + "source_group:chongqing-osm-roads-2021:license_terms_unverified", + "source_group:chongqing-osm-roads-2021:retention_policy_missing", + "source_group:chongqing-osm-roads-2021:access_policy_missing", + "source_group:chongqing-osm-roads-2021:odbl_attribution_policy_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8298af4d695e701f1346e5792f9bca491c0be490d8a3c0cf20baa5eed941da7a", + "data_classification": "restricted_local_open_data_copy", + "data_domains": [ + "transport_network", + "vector" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 8, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "02", + "size_bytes": 25416298, + "source_group_id": "chongqing-osm-roads-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-clcd-2020:owner_reference_missing", + "source_group:chongqing-clcd-2020:license_terms_unverified", + "source_group:chongqing-clcd-2020:retention_policy_missing", + "source_group:chongqing-clcd-2020:access_policy_missing", + "source_group:chongqing-clcd-2020:classification_lineage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "e3bd232f24c41c6b77e067690b4c70d68b20da6600f8be59c60de5868077420e", + "data_classification": "restricted_local_remote_sensing", + "data_domains": [ + "land_cover", + "remote_sensing", + "raster" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "tfw": 1, + "tif": 1, + "xlsx": 1, + "xml": 2 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "03", + "size_bytes": 9879957, + "source_group_id": "chongqing-clcd-2020" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-central-buildings-2021:owner_reference_missing", + "source_group:chongqing-central-buildings-2021:license_terms_unverified", + "source_group:chongqing-central-buildings-2021:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:access_policy_missing", + "source_group:chongqing-central-buildings-2021:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "a9343b5f8971c731fefee08864fd6b586d989e4f52d0cca584d6d70152de5039", + "data_classification": "restricted_local_built_environment", + "data_domains": [ + "buildings", + "urban_form", + "vector" + ], + "extension_counts": { + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "04", + "size_bytes": 22006362, + "source_group_id": "chongqing-central-buildings-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-historic-districts:owner_reference_missing", + "source_group:chongqing-historic-districts:license_terms_unverified", + "source_group:chongqing-historic-districts:retention_policy_missing", + "source_group:chongqing-historic-districts:access_policy_missing", + "source_group:chongqing-historic-districts:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "4ea511bb915b54fd21bd1addedad93cfa65de6f87a2940bdbddb0cf6f95b6f96", + "data_classification": "restricted_local_cultural_planning", + "data_domains": [ + "cultural_heritage", + "planning_constraints", + "vector" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 8, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "05", + "size_bytes": 372603, + "source_group_id": "chongqing-historic-districts" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:bishan-planning-materials:owner_reference_missing", + "source_group:bishan-planning-materials:license_terms_unverified", + "source_group:bishan-planning-materials:retention_policy_missing", + "source_group:bishan-planning-materials:access_policy_missing", + "source_group:bishan-planning-materials:planning_sensitivity_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "1d2ff573f09b991fffb79dbc85f630e0176192b3a98ab86aba1706fef414877c", + "data_classification": "highly_restricted_planning", + "data_domains": [ + "land_use", + "planning", + "documents", + "vector", + "tables" + ], + "extension_counts": { + "[no_ext]": 4, + "atx": 34, + "cpg": 18, + "dbf": 28, + "doc": 6, + "docx": 6, + "dwg": 17, + "dwl": 2, + "dwl2": 2, + "freelist": 3, + "gdbindexes": 15, + "gdbtable": 17, + "gdbtablx": 17, + "horizon": 5, + "jpg": 34, + "pdf": 8, + "png": 2, + "prj": 28, + "sbn": 28, + "sbx": 28, + "shp": 28, + "shx": 28, + "spx": 4, + "xls": 5, + "xlsx": 12, + "xml": 33, + "zip": 1 + }, + "file_count": 413, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "07", + "size_bytes": 389617168, + "source_group_id": "bishan-planning-materials" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-district-population-2021:owner_reference_missing", + "source_group:chongqing-district-population-2021:license_terms_unverified", + "source_group:chongqing-district-population-2021:retention_policy_missing", + "source_group:chongqing-district-population-2021:access_policy_missing", + "source_group:chongqing-district-population-2021:statistics_vintage_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8ac775d79db7c98fa25e0f697ee5f86ee8e35a095a0fd906890251c664684757", + "data_classification": "restricted_aggregate_population", + "data_domains": [ + "population", + "statistics", + "table" + ], + "extension_counts": { + "xlsx": 1 + }, + "file_count": 1, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "08", + "size_bytes": 13036, + "source_group_id": "chongqing-district-population-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:gaode-poi-2024:owner_reference_missing", + "source_group:gaode-poi-2024:license_terms_unverified", + "source_group:gaode-poi-2024:retention_policy_missing", + "source_group:gaode-poi-2024:access_policy_missing", + "source_group:gaode-poi-2024:contact_field_privacy_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "a5def66308dc5d3bee6a7be235fce935fd5b8a0182d6669551493f392d0e8918", + "data_classification": "highly_restricted_commercial_location", + "data_domains": [ + "poi", + "commercial_location", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "09", + "size_bytes": 233667334, + "source_group_id": "gaode-poi-2024" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:baidu-aoi-2024:owner_reference_missing", + "source_group:baidu-aoi-2024:license_terms_unverified", + "source_group:baidu-aoi-2024:retention_policy_missing", + "source_group:baidu-aoi-2024:access_policy_missing", + "source_group:baidu-aoi-2024:contact_field_privacy_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "09e1d3034316e936300607637ae3f220444d89004629aa9e2ea8ccfefbe6100a", + "data_classification": "highly_restricted_commercial_location", + "data_domains": [ + "aoi", + "commercial_location", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "10", + "size_bytes": 17006652, + "source_group_id": "baidu-aoi-2024" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:unicom-commuting-2023:owner_reference_missing", + "source_group:unicom-commuting-2023:license_terms_unverified", + "source_group:unicom-commuting-2023:retention_policy_missing", + "source_group:unicom-commuting-2023:access_policy_missing", + "source_group:unicom-commuting-2023:privacy_impact_assessment_missing", + "source_group:unicom-commuting-2023:grid_geometry_dictionary_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8c494e7fce9e43208ab8ec09872cf0febd48bebe5b425629b71278ad285fae32", + "data_classification": "highly_restricted_aggregate_mobility", + "data_domains": [ + "mobility", + "population", + "aggregate_signaling", + "table" + ], + "extension_counts": { + "csv": 1 + }, + "file_count": 1, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "11", + "size_bytes": 69667, + "source_group_id": "unicom-commuting-2023" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:baidu-search-index-2023:owner_reference_missing", + "source_group:baidu-search-index-2023:license_terms_unverified", + "source_group:baidu-search-index-2023:retention_policy_missing", + "source_group:baidu-search-index-2023:access_policy_missing", + "source_group:baidu-search-index-2023:commercial_terms_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "87db57d0821164492800c4271f817165ce884d6b261b46595c6e3f1e6b4ee1cf", + "data_classification": "highly_restricted_commercial_activity", + "data_domains": [ + "search_activity", + "intercity_flow", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "12", + "size_bytes": 201462, + "source_group_id": "baidu-search-index-2023" + } + ], + "status": "blocked_pending_source_governance" +} diff --git a/docs/roadmap.md b/docs/roadmap.md index f8813222..3c88db78 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,9 +417,9 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销；ADR-053 将生产 OIDC federation、双 provider integration/workload identity、最小权限、TLS/mTLS、持久 Gravitino catalog、tenant isolation、运营责任和新鲜 protected attestation 冻结为 fail-closed readiness contract。Gravitino `1.3.0` 镜像只发现 Basic IdP，不假设 native OIDC；当前 profile 仍有 40 个外部 blockers 且未提交真实 attestation。M3-27 已将 identity/object-store 两个 profile 与两份 protected attestation 的全部 85 个 blocker 精确且唯一分配给 16 个无环 owner decision groups，但不替 owner 选择 IdP、provider、account、bucket、region、KMS 或运行责任，所有 group 仍为 `unresolved`。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver、`production_identity_gate_passed` 与 `production_ready` 仍为 false。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销；ADR-053 将生产 OIDC federation、双 provider integration/workload identity、最小权限、TLS/mTLS、持久 Gravitino catalog、tenant isolation、运营责任和新鲜 protected attestation 冻结为 fail-closed readiness contract。Gravitino `1.3.0` 镜像只发现 Basic IdP，不假设 native OIDC；当前 profile 仍有 40 个外部 blockers 且未提交真实 attestation。M3-27 已将 identity/object-store 两个 profile 与两份 protected attestation 的全部 85 个 blocker 精确且唯一分配给 16 个无环 owner decision groups，但不替 owner 选择 IdP、provider、account、bucket、region、KMS 或运行责任，所有 group 仍为 `unresolved`。M3-28 又以全量重庆真实源启动 AR-2：固定 468,462,251-byte 原 ZIP、584-file/700,610,744-byte 解压工作集、11 个 source groups 与 16 个 metadata profiles，但因 6 个 archive entry 已变化、52 个解压新增文件及 57 个治理 blocker，只允许 metadata profiling，不允许 content admission。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver、`production_identity_gate_passed` 与 `production_ready` 仍为 false。 -M3-22 至 M3-25 已把一份真实重庆 20-feature EPSG:4490 slice 从受授权 Spark/Sedona + JDBC/S3 Iceberg ingestion，推进到保留 7 天的 staging material、原子 GDA Control `ResourceVersion + 2 Artifacts + QualityResult + LineageEvent` 晋级、数据库裁决的 `succeeded@3` 和保留运行时的受控 restart continuity。M3-24 已持久化完整 execution-plan/PolicyDecision/Approval Artifacts，回读真实 DolphinScheduler `SUCCESS`，由独立 evaluator 重开 Parquet 并重算九项空间质量与 row fingerprint；M3-25 又按 PostgreSQL -> MinIO -> Gravitino -> GDA Control 顺序重启同一运行时，证明稳定 namespace/StatefulSet/Service/PVC/container/volume 身份不变、Pod/PID 轮换，且 Iceberg snapshot/Parquet/Gravitino projection/GDA ledger facts SHA 与 `succeeded@3` 精确 replay 均不漂移、不新增事实。M3-26 进一步把该真实 predecessor 与 production identity/object-store gates 组合为受保护重执行准入，强制两份 attestation 绑定同一 source revision，并禁止本地 retained material 直推；M3-27 又将 85 个 blocker 精确映射为 16 个 owner decision groups，固化依赖、允许/禁止边界、profile path、required artifact 和 protected verification command。当前两个 profile 虽结构有效，但所有 group 仍未决，所以 `ready_for_protected_reexecution=false`，没有调度或 provider 写入授权。生产 owner 决策与 identity/storage/tenant attestation、常驻 scheduler/executor、独立故障域、backup/PITR、production restart recovery、staging scale 和完整 Spark/Flink conformance 仍是下一门槛。边界见 [ADR-070](architecture-decisions/adr-070-retained-real-feature-terminal-success.md)、[ADR-071](architecture-decisions/adr-071-retained-real-feature-restart-recovery.md)、[ADR-072](architecture-decisions/adr-072-protected-real-feature-reexecution-gate.md) 与 [ADR-073](architecture-decisions/adr-073-protected-profile-owner-decision-packet.md)。 +M3-22 至 M3-25 已把一份真实重庆 20-feature EPSG:4490 slice 从受授权 Spark/Sedona + JDBC/S3 Iceberg ingestion，推进到保留 7 天的 staging material、原子 GDA Control `ResourceVersion + 2 Artifacts + QualityResult + LineageEvent` 晋级、数据库裁决的 `succeeded@3` 和保留运行时的受控 restart continuity。M3-24 已持久化完整 execution-plan/PolicyDecision/Approval Artifacts，回读真实 DolphinScheduler `SUCCESS`，由独立 evaluator 重开 Parquet 并重算九项空间质量与 row fingerprint；M3-25 又按 PostgreSQL -> MinIO -> Gravitino -> GDA Control 顺序重启同一运行时，证明稳定 namespace/StatefulSet/Service/PVC/container/volume 身份不变、Pod/PID 轮换，且 Iceberg snapshot/Parquet/Gravitino projection/GDA ledger facts SHA 与 `succeeded@3` 精确 replay 均不漂移、不新增事实。M3-26 进一步把该真实 predecessor 与 production identity/object-store gates 组合为受保护重执行准入，强制两份 attestation 绑定同一 source revision，并禁止本地 retained material 直推；M3-27 又将 85 个 blocker 精确映射为 16 个 owner decision groups，固化依赖、允许/禁止边界、profile path、required artifact 和 protected verification command。当前两个 profile 虽结构有效，但所有 group 仍未决，所以 `ready_for_protected_reexecution=false`，没有调度或 provider 写入授权。M3-28 的全量重庆 admission evidence 是另一条 AR-2 源基线，不继承 20-feature slice 的授权或成功状态；它只证明源文件完整盘点、内容指纹与 metadata profiling，不能触发 scheduler/provider。生产 owner 决策与 identity/storage/tenant attestation、source governance、常驻 scheduler/executor、独立故障域、backup/PITR、production restart recovery、staging scale 和完整 Spark/Flink conformance 仍是下一门槛。边界见 [ADR-070](architecture-decisions/adr-070-retained-real-feature-terminal-success.md)、[ADR-071](architecture-decisions/adr-071-retained-real-feature-restart-recovery.md)、[ADR-072](architecture-decisions/adr-072-protected-real-feature-reexecution-gate.md)、[ADR-073](architecture-decisions/adr-073-protected-profile-owner-decision-packet.md) 与 [ADR-074](architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md)。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） @@ -441,6 +441,7 @@ M3-22 至 M3-25 已把一份真实重庆 20-feature EPSG:4490 slice 从受授权 交付： +- 真实源 admission contract：以不可变 archive checksum、解压 payload fingerprint、source-group manifest、metadata profile 和治理 blocker 建立准入基线；证据不得含源 payload、绝对路径、记录值或 geometry，profiling 不等于 content admission。 - SourceDefinition、CredentialReference、SourceCapability、SyncDefinition/Version、SyncRun、Cursor/Watermark、SchemaDriftEvent 和 Reconciliation。 - 数据库、对象存储/空间文件、HTTP/STAC 三类代表 source 的连接、凭据、连通、发现、preview、profile 和 owner 登记。 - 全量/增量微批的 Append/Overwrite/Merge 策略，以及至少一个真实 CDC 或事件流 source 通过 Flink 写入版本化 Bronze；覆盖 watermark/offset、checkpoint、迟到/乱序、源端删除、幂等、对账、重放和失败恢复。 @@ -688,7 +689,7 @@ Golden checks 至少覆盖： 3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。 4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b 本地与跨集群恢复、M2c metrics/OTel 故障演练、M2d production readiness contracts，以及 M3-1 至 M3-25 projection、provider identity/interoperability、Active Metadata、真实 feature ingestion、原子 ledger promotion、retained terminal success 和同一保留 authority 的本地进程重启连续性已验证；M3-26 已建立真实 predecessor 的 protected re-execution 组合门禁，M3-27 已把 85 个外部 blockers 收敛为 16 个可分派、可校验但仍未决的 owner decision groups。下一步必须由 packet 指定的 owner 批准并物化 production identity/object-store profiles，在同一 source revision 生成受保护 identity/storage/tenant attestations；随后才可另行授权常驻 scheduler/executor 与持久 catalog/control/storage，做 fresh protected ingestion，并验证 backup/PITR、独立故障域、production restart recovery、staging scale、完整 Spark/Flink conformance 和真实告警/runbook。本地 retained material 不得直接晋级，也不计入生产退出门。 5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入；不再开发新的 lease/queue/scheduler。 -6. 冻结首条地类图斑数据、标准版本、敏感级别、owner、SLO 和 golden result。 +6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline；下一步补齐解压派生 provenance，并由 owner 决定首条地类图斑源的 license、retention、access、privacy/sensitivity、标准版本、SLO 和 golden result，未获批准前不得 content admission。 7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles；以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。 8. 实现跨 profile 的 Raw -> ODS -> DIM/DWD -> DWS -> ADS 通用生产、质量、发布、回滚和 golden equivalence。 9. 建立 DataProductBlueprint、模型版本和 Visual/SQL/Notebook 共用 definition 的 Build 工作台，打通 preview、test、publish、approval 和 rollback。 @@ -729,7 +730,7 @@ AR-4 parity/control gate 退出前暂停以下主线扩张： |---|---|---| | AR-0 Architecture/Schema/Runtime Truth Freeze | `in_progress` | 全环境 schema/config fingerprint、迁移 fail-closed、事实清单、storage/compute/GIS serving provider profile/capability、ADR-017 benchmark、owner/SLO 和首条数据/服务验收集冻结 | | AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric 本地 recovery/metrics/policy/identity/interoperability、真实 feature ingestion 与临时 GDA Control 原子 output promotion 已验证；下一证据是可保留 staging material、完整授权/成功/独立质量 provenance、source host/cluster 外 recovery、持久 metrics/TLS/tenant/alert/SLO、OIDC、NetworkPolicy、升级回滚/registry provenance 与无双写验收 | -| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `planned` | 三类代表源、`DriveTransfer` 云盘客户端和大文件恢复通过统一控制面；默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | +| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 已完成全量重庆真实源的 path-free、content-addressed、metadata-only admission baseline；下一证据是解压派生 provenance 与 owner/license/retention/access/privacy 决策，之后才可建立 Landing authority，并让三类代表源、`DriveTransfer`、默认湖仓、轻量 profile 与 Azure adapter 通过统一控制面及 Raw -> ADS 验收 | | AR-3 Data Product Engineering + Governance Workbench | `planned` | Blueprint、模型、Visual/SQL/Notebook、DataOps CI/CD、质量/安全/审批共用 definition 和产品生命周期 | | AR-4 Asset/GIS Service/Spatial Experience Operations | `planned` | Service Control Plane、Features/Tiles/MVT/COG/STAC/export 及条件 legacy OGC/3D/EDR provider、Gateway/权限/缓存、原子切换/回滚、Discover/Operate/Govern 和无 LLM 多入口通过 conformance/parity/control gate | | AR-5 AgentOps Runtime + UX Uplift | `planned` | DataOps parity/control 通过；Agent bundle eval、deployment、online observation、incident/rollback 和 uplift gate | diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 5fecf1df..21178af9 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-31 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-25 retained real-feature restart continuity 已验证，M3-26 protected re-execution 组合门禁已建立，M3-27 已把 85 个 blockers 收敛为 16 个仍未决的 owner decision groups；生产 policy/tenant isolation、生产 identity/object-store attestation、常驻 production consumer/scheduler/executor、backup/PITR、独立故障域、production restart recovery、scale/conformance 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-25 retained real-feature restart continuity 已验证，M3-26 protected re-execution 组合门禁已建立，M3-27 已把 85 个 blockers 收敛为 16 个仍未决的 owner decision groups；AR-2 M3-28 已建立重庆全量真实源的 metadata-only admission baseline，但 content admission、Landing authority 和 ingestion 均未授权；生产 policy/tenant isolation、生产 identity/object-store attestation、常驻 production consumer/scheduler/executor、backup/PITR、独立故障域、production restart recovery、scale/conformance 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-protected-profile-decision-packet` +适用分支：`feat/ar2-chongqing-real-source-admission` ## 判定规则 @@ -21,7 +21,7 @@ | 部署配置策略 | Compose/K8s/进程环境；`platform_truth.CONFIG_SPECS` 定义关键类型与策略；DolphinScheduler worker 与 Active Metadata consumer 均有默认零副本、外部 ConfigMap/Secret 驱动的 Kustomize 模板和静态 validator，前者另有 staging activation preflight | `.env` 仅补默认；脱敏 snapshot、Secret key attestation、未扩容 Deployment 和 `ready_for_activation` 都是观测/模板 | 版本化 DeploymentProfile + secret reference；部署环境始终优先；模板或 preflight 通过都不等于环境已启用 | Platform/SRE/Security | AR-0，部分实现；worker 模板/preflight 本地已验证 | | 环境发布与晋级 | 本地 candidate/registry/provenance/release/live 合同已绑定 publisher、verifier、OCI 和 manifest identity；canonical `main@0182406`、archive refs、三组 active ruleset 与 `staging-provenance` protected environment 已建立，但尚无成功 publisher/verifier 或 deployment；M3-26 又把真实重庆 predecessor 与 identity/object-store production gates 组合为 source-revision-aligned protected re-execution preflight，M3-27 将其 85 个 blocker 唯一分配给 16 个 owner decision groups | 旧 mainline、feature branch、CI artifact、JSON、离线 report、合成 `verified_for_staging_apply`、M3-26 pending decision 和 M3-27 unresolved packet 都不能单独成为发布或执行权威；本地 retained material 禁止直接晋级 | 由受保护 environment 的 DeploymentRevision 绑定 OCI、provenance artifact、release manifest 与全部 live verdict；owner 必须先把批准值物化到 production profiles 并生成同 revision attestations，fresh protected ingestion 还须另有 tenant-bound PolicyDecision/Approval 与 scheduler command | Platform/SRE/Security/Repository Owner | AR-1 mainline 治理已恢复；M3-27 groups `unresolved` -> owner 批准 profiles/attestations -> M3-26 preflight 复评 -> 首次 GHCR publish/verify -> 真实 staging | | 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；Active Metadata consumer 登记为 `activation_request_staging_only`，其 deployment 默认为 0 replicas；M3-24 terminal-success 与 M3-25 restart-recovery rehearsal 及此前 Metadata Fabric rehearsals 均登记为 `local_verification_only` | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy/catalog/interoperability/failure/outbox/consumer/authorization/delivery/projection/binding/ingestion/promotion/terminal/restart evidence | PlatformRun ledger 唯一登记最终状态；M3-24 的 Run 已由数据库 evidence gate 裁决为 `succeeded@3`，M3-25 只证明同一单主机 retained namespace/PVC/MinIO-Iceberg/control DB 的进程重启连续性；两者均不得变成生产控制器、catalog/storage/output、tenant-isolation 或 production recovery 权威 | Platform Architecture | AR-1 adapter/worker、M3-15 至 M3-25 本地控制链、retained terminal success 与进程重启连续性已验证；常驻受保护 scheduler/executor、生产 storage/control、backup/PITR 与独立故障域待接入 | -| 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一 | 临时上传、下载缓存、预览文件 | Landing object 以 immutable URI + checksum + retention 为权威；本地 scratch 可删除 | Data Platform | AR-2 | +| 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一；M3-28 已把重庆原 ZIP `2043b60c...bbca` 与 584-file 解压工作集 `e7e81e4f...8ea6` 分别内容寻址，确认 archive 自身完整但解压集有 6 个变化项和 52 个新增文件 | 临时上传、下载缓存、预览文件、研究 inventory 和 M3-28 metadata evidence；它们都不是 admitted content 或 Landing authority | Landing object 以 immutable URI + checksum + retention 为权威；源 owner/license/retention/access/privacy 与解压派生 provenance 全部批准后，必须 fresh protected ingest，本地 scratch 可删除且不得直推 | Data Platform/Governance | AR-2 M3-28 metadata admission baseline 已验证；57 blockers 未决，content admission/ingestion 未授权 | | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；M3-14 将新 ResourceVersion 与变化事件同事务创建；M3-23 建立 exact-replay output bundle promoter，M3-24 又对仍可读的 retained material 原子追加同类 bundle 并完成终局 | UI catalog、search index、STAC、Active Metadata delivery state | GDA ledger 管身份与版本绑定；authority Resource 必须预先存在，旧行不得伪装成实时事件或部分 promotion；OpenMetadata 管治理目录，Gravitino 管技术对象映射；本地 retained version 不自动成为生产版本 | Metadata Platform | AR-1 gateway/M3-14/M3-24 retained GDA Control 已验证 -> 生产写入口与 storage authority 切换待验收 | @@ -71,6 +71,7 @@ 26. M3-25 不重新 ingest、不创建新 ResourceVersion/Run，也不改写 M3-24 verdict；它只绑定 M3-24 evidence/retention ID，按 PostgreSQL、MinIO、Gravitino、GDA Control 顺序重启同一保留运行时。三组 Pod UID 与 control PID/StartedAt 必须轮换，namespace/StatefulSet/Service/PVC/image/control container/volume 必须不变；Iceberg inventory/metadata/snapshot、Parquet 九项质量、Gravitino projection 和 GDA Control facts SHA 在重启前后与 exact terminal replay 后必须一致且零新增。该结果只证明单开发主机本地进程重启连续性，不证明 backup/PITR、独立故障域、production restart recovery 或 production readiness。 27. M3-26 将 checked M3-25 evidence file/evidence SHA、真实 output ResourceVersion/content、Iceberg/Parquet/quality/control facts 与 ADR-053 identity gate、ADR-057 object-store gate 组合为单一 protected re-execution preflight。两份新鲜 protected attestation 必须各自通过既有完整 gate，且绑定同一 source revision；本地 retained material 永远不可直推，preflight 通过也只表示可另行申请 fresh protected ingestion，不创建 PolicyDecision/Approval、scheduler command、Run 或 provider mutation。当前 profiles 结构有效但有 identity 40 + object-store 43 项未决输入及两份缺失 attestation，共 85 blockers，故 `ready_for_protected_reexecution=false`，所有 production claims 仍为 false。 28. M3-27 固定绑定 M3-26 evidence file/decision/contract SHA，将 identity profile 40、object-store profile 43 和两份 attestation blocker 精确且唯一分配到 16 个无环 owner decision groups。每组冻结 owner role、依赖、profile path、允许/禁止边界、required artifact 与 protected verification command；所有 status 均为 `unresolved`，packet 不记录生产选择、不含凭据、不创建授权或 provider mutation，也不降低 85 blocker 计数。 +29. M3-28 扫描重庆全量真实源但只提交 path-free metadata evidence：原 ZIP 为 533 files/468,462,251 bytes，`01数据样例` archive scope 为 532 files/694,147,946 uncompressed bytes，当前解压集为 584 files/700,610,744 bytes，覆盖 11 个 source groups 和 16 个 asset profiles。archive CRC 完整，但 archive/extracted multiset 不等价（526 exact、6 modified、0 missing、52 additional），故 extraction derivation provenance 与每组 owner/license/retention/access 及领域治理项共 57 个 blockers 必须保留。该 evidence 不含原始文件、绝对路径、记录值或 geometry，不创建 Landing/ResourceVersion/Run/authorization/scheduler/provider 事实，所有 content/publication/production claims 为 false。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -121,11 +122,13 @@ - Metadata Fabric M3-25 已在同一 retention ID 和 expiry 内按 PostgreSQL -> MinIO -> Gravitino -> GDA Control 顺序完成进程重启。三组 Pod UID 与 control PID 均轮换，namespace/StatefulSet/Service/PVC/image/control container/volume 均保持；object inventory SHA `de4a0efed9fdb68f0019b843377f6c8de71664de955130d0dd38e99eccdb8034`、snapshot `8034081021802585202`、94,603-byte Parquet body/row-set SHA、九项 20-count 质量、Gravitino projection SHA `f30feb94a5a8280597f331a7f965762bfabb9b82397af49dda05b61ce00bbb1e` 与 GDA Control facts SHA `5c0b8a58729e551c250b0410bcebdfe3f019f215a50b20d503f094fe1562d8b5` 前后不变；terminal replay 为 `created=false`，ledger 计数零新增。contract/evidence SHA 为 `83ed15ae4eed85e0c261c2b3a04ea2ad559f3deb7b86c7c2f2dedd0cf28d23d0` / `1b5a5ceeadee88868bab6237b3f3280c8b13793cc54193592fec7dbbfdd4e8a6`；backup/PITR、独立故障域、production restart recovery 与 `production_ready` 均为 `false`。 - Metadata Fabric M3-26 已把 M3-25 evidence file/evidence SHA、20-feature output ResourceVersion/content、snapshot/object/Parquet/row-set/control facts 与 production identity/object-store profile fingerprints 组合为 deterministic protected re-execution contract。contract/decision SHA 为 `39411248b37b7d8d43ac7ad37737de15d6b6d4c5e4feb2088f0a87cec888b5f9` / `39246eacdd1793f23aecb71195cc4c9d8c63d7125aad8cd9bbb59a96c588cd73`；当前 decision 自验证通过，但 85 个外部 blockers 和缺失 protected attestations 使 status 保持 `blocked_pending_protected_attestation`。本地 retained material 不可直推，fresh protected ingestion、scheduler submission、provider mutation、production ingestion 与 `production_ready` 均未获授权或验证。 - Metadata Fabric M3-27 已把 M3-26 的 85 个 blocker 收敛为 16 个 owner decision groups：identity profile `40`、object-store profile `43`、identity/object-store attestation 各 `1`，全集相等且每项唯一、依赖图无环。packet SHA 为 `b23e1becafa0c91541a653d467e7959107008d4dc41576018b57b564e6b46a36`；所有 group 均为 `unresolved`，全部 production/execution claims 为 `false`。它只是外部决策工作包，不是批准、attestation、调度或生产 authority。 +- AR-2 M3-28 已对重庆全量真实源生成 metadata-only admission evidence：archive SHA `2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca`，extracted payload SHA `e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6`，11 个 source groups、16 个 profiles、57 个 blockers。旧研究报告的村规范围已纠正为 28 个 Shapefile layers、20 个非空 layers 和 8,050 features；31 是全 root 的 Shapefile 数。evidence/file SHA 为 `a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1` / `9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83`；source content、publication、scheduler、provider、production ingestion 和 readiness 均为 `false`。 ## 下一验收证据 - M3-21 空表 metadata promotion、M3-22 临时真实 feature ingestion、M3-23 临时 ledger promotion、M3-24 retained terminal success、M3-25 本地进程重启连续性与 M3-26 pending preflight 都不计入生产对象存储、持久生产 authority、production recovery 或 ingestion 退出门，历史 candidate/evidence 保持不变； - M3-27 已将下一次真实数据执行前的 owner 工作顺序冻结：先完成 identity federation/provider binding/TLS/catalog/tenancy/operations 与 object-store provider/workload identity/transport/KMS/durability/tenancy/operations groups，再由两个 protected environment 对同一 source revision 生成新鲜 attestations，并由 M3-26 复评清空 85 blockers； +- M3-28 之后先取得 archive/extracted 差异的派生 provenance，并逐组批准 owner/license/retention/access/privacy/commercial/sensitivity 等 57 个 blockers；批准结果必须进入正式 source/governance authority，不能直接改写 checked evidence。随后才可为选定源创建 immutable Landing object、ResourceVersion 和 tenant-bound fresh ingestion authorization； - M3-26 通过后仍须另行创建 tenant-bound PolicyDecision/Approval 和 scheduler command，做 fresh protected ingestion。任何本地 retained material 复制/直推或跨 revision attestation 组合都必须拒绝； - preflight 通过后再部署常驻 scheduler/executor、持久 catalog/control/storage，验证 backup/PITR、独立故障域、production restart recovery、staging scale 与 Spark/Flink conformance； - 完成首次 application subject publish 与 protected verifier run；当前 mainline、archive refs、ruleset、required reviewer、禁止 bypass 和 environment enable variable 已配置并复核； diff --git a/scripts/chongqing-real-source-admission.sh b/scripts/chongqing-real-source-admission.sh new file mode 100755 index 00000000..90ce3dec --- /dev/null +++ b/scripts/chongqing-real-source-admission.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +common_git_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +shared_root="" +if [ -n "$common_git_dir" ]; then + shared_root="$(cd "$common_git_dir/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$repo_root/.venv/bin/python" ]; then + PYTHON="$repo_root/.venv/bin/python" +elif [ -n "$shared_root" ] && [ -x "$shared_root/.venv/bin/python" ]; then + PYTHON="$shared_root/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$repo_root" +exec "$PYTHON" -m data_agent.chongqing_real_source_admission "$@"