- OpsScript Gate Runtime (v0.8.3)
+ OpsScript Gate Runtime (v0.9.0)
1 DISTRO FAILED
@@ -544,7 +544,7 @@
steps:
- uses: actions/checkout@v4
- name: Validate scripts across distributions
- uses: Mresyzz/opsscript-gate@v0.8.3
+ uses: Mresyzz/opsscript-gate@v0.9.0
with:
script-path: ./install.sh
shell: auto
@@ -612,7 +612,7 @@ argparse.ArgumentParser:
default=DEFAULT_TIMEOUT,
help=f"Hard timeout in seconds per container (default: {DEFAULT_TIMEOUT}s)",
)
+ run_parser.add_argument(
+ "--repeat",
+ type=int,
+ default=1,
+ help="Repeat each distribution run to detect flaky runtime behavior (default: 1)",
+ )
run_parser.add_argument(
"--format",
choices=["table", "markdown", "json", "sarif"],
@@ -162,6 +168,30 @@ def parse_packages_argument(packages_raw: list[str] | None) -> list[str]:
return validate_packages(packages)
+def run_checked_matrix(
+ args: argparse.Namespace, script_path: str, matrix: list[str]
+) -> RunReport | None:
+ """Run one script and normalize user-facing Docker/runtime errors."""
+ try:
+ return run_matrix(
+ script_path=script_path,
+ matrix=matrix,
+ timeout=args.timeout,
+ shell_mode=args.shell,
+ jobs=args.jobs,
+ mem_limit=args.mem_limit,
+ pids_limit=args.pids_limit,
+ network=args.network,
+ packages=args.packages,
+ repeat=args.repeat,
+ )
+ except DockerDaemonError as err:
+ sys.stderr.write(f"Docker Error: {err}\n")
+ except Exception as err:
+ sys.stderr.write(f"Unexpected Error: {err}\n")
+ return None
+
+
def main(argv: list[str] | None = None) -> int:
"""Main CLI entrypoint."""
parser = build_parser()
@@ -228,7 +258,7 @@ def main(argv: list[str] | None = None) -> int:
args.preset = None
if "--preset" in explicit:
args.matrix = None
- for key in ("jobs", "timeout", "pids_limit", "max_scripts"):
+ for key in ("jobs", "timeout", "pids_limit", "max_scripts", "repeat"):
value = getattr(args, key)
if value is not None and value < 1:
raise ValueError(f"{key.replace('_', '-')} must be positive")
@@ -285,7 +315,8 @@ def main(argv: list[str] | None = None) -> int:
if args.dry_run:
plan = {"dry_run": True, "scripts": target_scripts, "matrix": matrix,
"executions": len(target_scripts) * len(matrix), "shell": args.shell,
- "network": args.network, "packages": args.packages, "timeout": args.timeout}
+ "network": args.network, "packages": args.packages, "timeout": args.timeout,
+ "repeat": args.repeat}
if changed_selection:
plan["changed_since"] = args.changed_since
output = json.dumps(plan, indent=2) if args.format == "json" else (
@@ -293,7 +324,7 @@ def main(argv: list[str] | None = None) -> int:
+ "\n".join(f" {path}" for path in target_scripts)
+ f"\nImages: {', '.join(matrix)}\nContainer executions: {plan['executions']}"
+ (f"\nPackages: {', '.join(args.packages)}" if args.packages else "")
- + f"\nShell: {args.shell} | Network: {args.network} | Timeout: {args.timeout}s"
+ + f"\nShell: {args.shell} | Network: {args.network} | Timeout: {args.timeout}s | Repeat: {args.repeat}"
)
return 0 if publish_output(output, args.output) else 1
@@ -305,23 +336,8 @@ def main(argv: list[str] | None = None) -> int:
if len(target_scripts) == 1:
script_path = target_scripts[0]
- try:
- report = run_matrix(
- script_path=script_path,
- matrix=matrix,
- timeout=args.timeout,
- shell_mode=args.shell,
- jobs=args.jobs,
- mem_limit=args.mem_limit,
- pids_limit=args.pids_limit,
- network=args.network,
- packages=args.packages,
- )
- except DockerDaemonError as err:
- sys.stderr.write(f"Docker Error: {err}\n")
- return 1
- except Exception as err:
- sys.stderr.write(f"Unexpected Error: {err}\n")
+ report = run_checked_matrix(args, script_path, matrix)
+ if report is None:
return 1
# Format report output
@@ -350,23 +366,8 @@ def main(argv: list[str] | None = None) -> int:
multi_start = time.perf_counter()
for script_path in target_scripts:
- try:
- report = run_matrix(
- script_path=script_path,
- matrix=matrix,
- timeout=args.timeout,
- shell_mode=args.shell,
- jobs=args.jobs,
- mem_limit=args.mem_limit,
- pids_limit=args.pids_limit,
- network=args.network,
- packages=args.packages,
- )
- except DockerDaemonError as err:
- sys.stderr.write(f"Docker Error: {err}\n")
- return 1
- except Exception as err:
- sys.stderr.write(f"Unexpected Error: {err}\n")
+ report = run_checked_matrix(args, script_path, matrix)
+ if report is None:
return 1
multi_reports[script_path] = report
diff --git a/src/opsscript_gate/config.py b/src/opsscript_gate/config.py
index cb3c1a6..1c8268f 100644
--- a/src/opsscript_gate/config.py
+++ b/src/opsscript_gate/config.py
@@ -14,8 +14,11 @@
"matrix": None, "preset": None, "jobs": None, "timeout": 60,
"shell": "posix", "mem_limit": "256m", "pids_limit": 128,
"network": "bridge", "packages": [], "exclude": [], "max_scripts": 20,
+ "repeat": 1,
}
+MAX_CONFIG_SIZE_BYTES = 256 * 1024
+
PACKAGE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9+._:-]*$")
@@ -39,6 +42,10 @@ def load_config(filename: str | None) -> dict:
if filename is None and not path.exists():
return {}
try:
+ if path.stat().st_size > MAX_CONFIG_SIZE_BYTES:
+ raise ValueError(
+ f"configuration exceeds {MAX_CONFIG_SIZE_BYTES // 1024} KiB limit"
+ )
config = json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, ValueError) as exc:
raise ValueError(f"Cannot load {path}: {exc}") from exc
@@ -48,7 +55,7 @@ def load_config(filename: str | None) -> dict:
if unknown:
raise ValueError(f"Unknown configuration keys: {', '.join(sorted(unknown))}")
for key, value in config.items():
- if key in {"jobs", "timeout", "pids_limit", "max_scripts"}:
+ if key in {"jobs", "timeout", "pids_limit", "max_scripts", "repeat"}:
if type(value) is not int or value < 1:
raise ValueError(f"{key} must be a positive integer")
elif key == "exclude":
@@ -83,7 +90,7 @@ def init_project() -> list[str]:
- uses: actions/checkout@v7
with:
persist-credentials: false
- - uses: Mresyzz/opsscript-gate@v0.8.3
+ - uses: Mresyzz/opsscript-gate@v0.9.0
with:
config: .opsscript-gate.json
""",
diff --git a/src/opsscript_gate/models.py b/src/opsscript_gate/models.py
index 535bd6c..fcf2e7e 100644
--- a/src/opsscript_gate/models.py
+++ b/src/opsscript_gate/models.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from dataclasses import asdict, dataclass, field
+from dataclasses import dataclass, field
from enum import Enum
from typing import Any
@@ -11,6 +11,7 @@ class DistroStatus(str, Enum):
FAIL = "FAIL"
TIMED_OUT = "TIMED_OUT"
ERROR = "ERROR"
+ FLAKY = "FLAKY"
class ShellMode(str, Enum):
@@ -56,6 +57,20 @@ class SingleResult:
output_snippet: str = ""
error_message: str | None = None
diagnostic: FailureDiagnostic | None = None
+ attempts: int = 1
+ passed_attempts: int = 1
+ failed_attempts: int = 0
+
+ def __post_init__(self) -> None:
+ if self.attempts < 1:
+ raise ValueError("attempts must be a positive integer")
+ if self.status != DistroStatus.PASS and self.attempts == 1:
+ # Preserve the old constructor shape while keeping failure counts honest.
+ if self.passed_attempts == 1 and self.failed_attempts == 0:
+ self.passed_attempts = 0
+ self.failed_attempts = 1
+ if self.passed_attempts + self.failed_attempts != self.attempts:
+ raise ValueError("passed_attempts + failed_attempts must equal attempts")
def to_dict(self) -> dict[str, Any]:
return {
@@ -66,6 +81,10 @@ def to_dict(self) -> dict[str, Any]:
"output_snippet": self.output_snippet,
"error_message": self.error_message,
"diagnostic": self.diagnostic.to_dict() if self.diagnostic is not None else None,
+ "attempts": self.attempts,
+ "passed_attempts": self.passed_attempts,
+ "failed_attempts": self.failed_attempts,
+ "flaky": self.status == DistroStatus.FLAKY,
}
diff --git a/src/opsscript_gate/reporter.py b/src/opsscript_gate/reporter.py
index 79134c4..c0c57c1 100644
--- a/src/opsscript_gate/reporter.py
+++ b/src/opsscript_gate/reporter.py
@@ -210,6 +210,10 @@ def format_terminal_table(report: RunReport, script_path: str = "") -> str:
if r.status == DistroStatus.PASS:
detail = "OK"
+ if r.attempts > 1:
+ detail = f"OK ({r.passed_attempts}/{r.attempts} attempts)"
+ elif r.status == DistroStatus.FLAKY:
+ detail = r.error_message or "Runtime was non-deterministic"
elif r.diagnostic:
line_suffix = f" (line {r.diagnostic.line})" if r.diagnostic.line else ""
detail = f"{r.diagnostic.message}{line_suffix}"
@@ -332,6 +336,8 @@ def format_markdown_compatibility_card(report: RunReport, script_path: str = "")
for r in report.results:
if r.status == DistroStatus.PASS:
status_icon = "✅ PASS"
+ elif r.status == DistroStatus.FLAKY:
+ status_icon = "⚠️ FLAKY"
elif r.status == DistroStatus.FAIL:
status_icon = "❌ FAIL"
elif r.status == DistroStatus.TIMED_OUT:
@@ -345,6 +351,8 @@ def format_markdown_compatibility_card(report: RunReport, script_path: str = "")
if r.status == DistroStatus.PASS:
raw_diag = "OK"
+ elif r.status == DistroStatus.FLAKY:
+ raw_diag = r.error_message or "Runtime was non-deterministic"
elif r.diagnostic:
line_str = f" (line {r.diagnostic.line})" if r.diagnostic.line else ""
clean_cmd = f"`{escape_inline_code(r.diagnostic.command)}`" if r.diagnostic.command else "command"
@@ -376,10 +384,12 @@ def format_markdown_compatibility_card(report: RunReport, script_path: str = "")
"| :--- | :---: | :---: | :--- |",
]
for r in report.results:
- icon = "✅" if r.status == DistroStatus.PASS else "❌"
+ icon = "✅" if r.status == DistroStatus.PASS else ("⚠️" if r.status == DistroStatus.FLAKY else "❌")
distro_cell = escape_markdown_table_cell(f"`{escape_inline_code(r.distro)}`")
if r.status == DistroStatus.PASS:
raw_detail = "OK"
+ elif r.status == DistroStatus.FLAKY:
+ raw_detail = r.error_message or "Runtime was non-deterministic"
elif r.diagnostic:
safe_cmd = escape_inline_code(r.diagnostic.command or "")
raw_detail = f"Missing `{safe_cmd}`"
@@ -469,6 +479,7 @@ def format_json(report: RunReport | MultiScriptReport) -> str:
DistroStatus.FAIL: ("OSG001", "Runtime compatibility failure"),
DistroStatus.TIMED_OUT: ("OSG002", "Runtime compatibility check timed out"),
DistroStatus.ERROR: ("OSG003", "Runtime compatibility check error"),
+ DistroStatus.FLAKY: ("OSG004", "Runtime compatibility check was flaky"),
}
diff --git a/src/opsscript_gate/runner.py b/src/opsscript_gate/runner.py
index 57c1cdc..2b41d43 100644
--- a/src/opsscript_gate/runner.py
+++ b/src/opsscript_gate/runner.py
@@ -747,24 +747,36 @@ def run_on_distro(
container.start()
- # Hard timeout monitoring with container.kill()
+ # Prefer Docker's blocking wait to avoid a tight synchronous reload loop.
+ # Test doubles and older clients that do not return a status dict fall back
+ # to the bounded polling path below.
timed_out = False
- while True:
- elapsed = time.perf_counter() - start_time
- if elapsed >= timeout:
+ wait_result = None
+ wait_fn = getattr(container, "wait", None)
+ if callable(wait_fn):
+ try:
+ wait_result = wait_fn(timeout=max(0.01, float(timeout)))
+ except Exception:
timed_out = True
- try:
- container.kill()
- except Exception:
- pass
- break
+ if not isinstance(wait_result, dict) and not timed_out:
+ while True:
+ elapsed = time.perf_counter() - start_time
+ if elapsed >= timeout:
+ timed_out = True
+ break
- container.reload()
- status_str = container.status.lower()
- if status_str in ("exited", "dead", "stopped"):
- break
+ container.reload()
+ status_str = container.status.lower()
+ if status_str in ("exited", "dead", "stopped"):
+ break
- time.sleep(poll_interval)
+ time.sleep(min(poll_interval, max(0.0, timeout - elapsed)))
+
+ if timed_out:
+ try:
+ container.kill()
+ except Exception:
+ pass
duration = time.perf_counter() - start_time
@@ -847,12 +859,16 @@ def run_matrix(
pids_limit: int = 128,
network: str = "bridge",
packages: Sequence[str] | None = None,
+ repeat: int = 1,
) -> RunReport:
"""
Run the compatibility check across all specified Linux distributions,
optionally running containers concurrently with ThreadPoolExecutor.
Results strictly preserve the original matrix order.
"""
+ if repeat < 1:
+ raise ValueError("repeat must be a positive integer")
+
try:
mode = ShellMode(shell_mode)
except ValueError:
@@ -900,20 +916,53 @@ def run_matrix(
results: list[SingleResult] = [None] * len(distro_list) # type: ignore
def _worker(index: int, distro_name: str) -> tuple[int, SingleResult]:
- res = run_on_distro(
- client=docker_client,
- script_path=script_path,
+ attempts: list[SingleResult] = []
+ for _ in range(repeat):
+ attempts.append(run_on_distro(
+ client=docker_client,
+ script_path=script_path,
+ distro=distro_name,
+ timeout=timeout,
+ shell_mode=shell_mode,
+ parsed_shebang=parsed_shebang,
+ mem_limit=mem_limit,
+ pids_limit=pids_limit,
+ network=network,
+ packages=package_list,
+ prepared_script_path=prepared_script,
+ ))
+
+ if repeat == 1:
+ return index, attempts[0]
+
+ passed = sum(result.status == DistroStatus.PASS for result in attempts)
+ failed = repeat - passed
+ representative = next((r for r in attempts if r.status != DistroStatus.PASS), attempts[-1])
+ if passed and failed:
+ status = DistroStatus.FLAKY
+ message = (
+ f"Runtime was non-deterministic: {passed}/{repeat} attempts passed "
+ f"and {failed}/{repeat} failed"
+ )
+ error_message = message if representative.error_message is None else (
+ f"{message}; {representative.error_message}"
+ )
+ else:
+ status = representative.status
+ error_message = representative.error_message
+
+ return index, SingleResult(
distro=distro_name,
- timeout=timeout,
- shell_mode=shell_mode,
- parsed_shebang=parsed_shebang,
- mem_limit=mem_limit,
- pids_limit=pids_limit,
- network=network,
- packages=package_list,
- prepared_script_path=prepared_script,
+ status=status,
+ exit_code=representative.exit_code,
+ duration=sum(r.duration for r in attempts),
+ output_snippet=representative.output_snippet,
+ error_message=error_message,
+ diagnostic=representative.diagnostic,
+ attempts=repeat,
+ passed_attempts=passed,
+ failed_attempts=failed,
)
- return index, res
try:
if effective_jobs == 1 or len(distro_list) <= 1:
diff --git a/tests/test_action_contract.py b/tests/test_action_contract.py
index d3844cc..f860f7e 100644
--- a/tests/test_action_contract.py
+++ b/tests/test_action_contract.py
@@ -7,11 +7,11 @@
def test_action_metadata_is_valid_and_inputs_are_wired():
text = (ROOT / "action.yml").read_text(encoding="utf-8")
assert "using: 'composite'" in text
- for name in ("config", "preset", "exclude", "max-scripts", "dry-run", "output", "changed-since", "packages"):
+ for name in ("config", "preset", "exclude", "max-scripts", "dry-run", "output", "changed-since", "packages", "repeat"):
assert f" {name}:" in text
run_text = text
for flag in ("--timeout", "--format", "--shell", "--mem-limit", "--pids-limit",
- "--network", "--packages", "--matrix", "--jobs", "--config", "--preset",
+ "--network", "--packages", "--matrix", "--jobs", "--config", "--preset", "--repeat",
"--max-scripts", "--output", "--exclude"):
assert flag in run_text
assert "json, or sarif" in text
diff --git a/tests/test_onboarding.py b/tests/test_onboarding.py
index 1525259..eab842b 100644
--- a/tests/test_onboarding.py
+++ b/tests/test_onboarding.py
@@ -98,6 +98,19 @@ def test_packages_are_forwarded_and_visible_in_dry_run(project, capsys):
assert plan["packages"] == ["curl", "ca-certificates", "tar"]
+def test_repeat_is_forwarded_and_visible_in_dry_run(project, capsys):
+ assert main(["run", "install.sh", "--dry-run", "--format", "json", "--repeat", "3"]) == 0
+ plan = json.loads(capsys.readouterr().out)
+ assert plan["repeat"] == 3
+
+
+def test_oversized_config_is_rejected(project):
+ Path(CONFIG_NAME).write_text("{" + "\"x\":1," * 50000 + "\"timeout\":60}")
+ with patch("opsscript_gate.cli.run_matrix") as run:
+ assert main(["run", "--dry-run"]) == 1
+ run.assert_not_called()
+
+
def test_packages_reject_offline_network(project):
with patch("opsscript_gate.cli.run_matrix") as run:
assert main(["run", "install.sh", "--packages", "curl", "--network", "none"]) == 1
@@ -169,7 +182,7 @@ def test_changed_discovery_preview_includes_revision(project, monkeypatch, capsy
@pytest.mark.parametrize("flags", [["--jobs", "0"], ["--timeout", "-1"],
- ["--pids-limit", "0"], ["--network", "host"], ["--matrix", ","],
+ ["--pids-limit", "0"], ["--repeat", "0"], ["--network", "host"], ["--matrix", ","],
["--matrix", "alpine", "--preset", "minimal"], ["--config", "missing.json"]])
def test_invalid_cli_before_execution(project, flags):
with patch("opsscript_gate.cli.run_matrix") as run:
diff --git a/tests/test_runner.py b/tests/test_runner.py
index 43b8dfc..98a6ad8 100644
--- a/tests/test_runner.py
+++ b/tests/test_runner.py
@@ -8,7 +8,7 @@
import pytest
from docker.errors import DockerException
-from opsscript_gate.cli import build_parser, main, parse_matrix_argument, parse_packages_argument
+from opsscript_gate.cli import build_parser, main, parse_matrix_argument, parse_packages_argument, publish_output
from opsscript_gate.discovery import discover_scripts, is_shell_script
from opsscript_gate.models import DistroStatus, FailureDiagnostic, MultiScriptReport, RunReport, ShellMode, SingleResult
from opsscript_gate.remediation import generate_remediation_hint, REMEDIATION_RULES
@@ -327,6 +327,29 @@ def test_reporter_terminal_table():
assert "cmd not found" in table_output
+def test_reporter_terminal_table_handles_empty_and_long_rows():
+ empty = format_terminal_table(RunReport())
+ assert "Distro" in empty
+ assert "Result: PASSED" in empty
+
+ long_error = SingleResult(
+ "alpine:3.20", DistroStatus.FAIL, 1, 0.1, error_message="x" * 200
+ )
+ table = format_terminal_table(RunReport([long_error]))
+ assert "x" * 47 + "..." in table
+
+
+def test_publish_output_writes_and_reports_os_errors(tmp_path, capsys):
+ target = tmp_path / "reports" / "result.txt"
+ assert publish_output("hello", str(target)) is True
+ assert target.read_text(encoding="utf-8") == "hello\n"
+ assert "hello" in capsys.readouterr().out
+
+ blocked = tmp_path / "file"
+ blocked.write_text("already a file", encoding="utf-8")
+ assert publish_output("nope", str(blocked / "child.txt")) is False
+
+
def test_reporter_github_summary():
r1 = SingleResult("debian:12-slim", DistroStatus.PASS, 0, 1.2)
r2 = SingleResult("ubuntu:22.04", DistroStatus.TIMED_OUT, None, 60.0, "hanging...", "Timed out")
@@ -1484,6 +1507,42 @@ def fake_run_on_distro(**kwargs):
assert [r.distro for r in report.results] == matrix
+def test_run_matrix_repeat_marks_flaky_runtime(tmp_path):
+ script_file = tmp_path / "flaky.sh"
+ script_file.write_text("#!/bin/sh\necho maybe\n", encoding="utf-8")
+ calls: dict[str, int] = {}
+
+ def fake_run_on_distro(**kwargs):
+ distro = kwargs["distro"]
+ calls[distro] = calls.get(distro, 0) + 1
+ if calls[distro] == 1:
+ return SingleResult(distro, DistroStatus.PASS, 0, 0.1)
+ return SingleResult(distro, DistroStatus.FAIL, 1, 0.2, error_message="intermittent failure")
+
+ with mock.patch("opsscript_gate.runner.run_on_distro", side_effect=fake_run_on_distro):
+ report = run_matrix(
+ script_path=str(script_file),
+ matrix=["alpine:3.20"],
+ repeat=2,
+ client=mock.MagicMock(),
+ )
+
+ result = report.results[0]
+ assert result.status == DistroStatus.FLAKY
+ assert result.attempts == 2
+ assert result.passed_attempts == 1
+ assert result.failed_attempts == 1
+ assert result.to_dict()["flaky"] is True
+ assert report.all_passed is False
+
+
+def test_run_matrix_rejects_non_positive_repeat(tmp_path):
+ script_file = tmp_path / "script.sh"
+ script_file.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
+ with pytest.raises(ValueError, match="repeat"):
+ run_matrix(str(script_file), matrix=["alpine:3.20"], repeat=0, client=mock.MagicMock())
+
+
def test_run_on_distro_resource_limits(tmp_path):
"""Verify mem_limit, pids_limit, and network_mode parameters are forwarded to Docker container."""
script_file = tmp_path / "test_limits.sh"