Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
- uses: psf/black@stable
- uses: actions/checkout@v6
- uses: astral-sh/ruff-action@v3
- run: ruff check
- run: ruff format --check

89 changes: 28 additions & 61 deletions main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#!/usr/bin/env python3

# Copyright (c) 2020-2021, Linus Groh <linusg@serenityos.org>
# Copyright (c) 2021, Marcin Gasperowicz <xnooga@gmail.com>
# Copyright (c) 2021, Idan Horowitz <idan.horowitz@serenityos.org>
Expand All @@ -21,12 +22,15 @@
import sys
import threading
import traceback

from argparse import ArgumentParser
from dataclasses import dataclass
from collections import Counter
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any, Callable, Optional
from typing import Any
from typing import Callable
from typing import Optional

from tqdm import tqdm

Expand Down Expand Up @@ -82,9 +86,7 @@ def run_streaming_script(
) -> subprocess.CompletedProcess:
def limit_memory():
if platform.system() != "Darwin":
resource.setrlimit(
resource.RLIMIT_AS, (memory_limit * 1024 * 1024, resource.RLIM_INFINITY)
)
resource.setrlimit(resource.RLIMIT_AS, (memory_limit * 1024 * 1024, resource.RLIM_INFINITY))

command = [
str(libjs_test262_runner),
Expand Down Expand Up @@ -127,11 +129,7 @@ def add_result(
exit_code: int = 0,
strict_mode: bool = False,
) -> None:
iteration_results.append(
TestRun(
test_file_paths[current_test], result, output, exit_code, strict_mode
)
)
iteration_results.append(TestRun(test_file_paths[current_test], result, output, exit_code, strict_mode))

new_results = []
while current_test < len(test_file_paths):
Expand All @@ -153,9 +151,7 @@ def add_result(
process_failed = True
process_result = e

test_results = [
part.strip() for part in process_result.stdout.strip().split("\0")
]
test_results = [part.strip() for part in process_result.stdout.strip().split("\0")]
have_stopping_result = False

while test_results:
Expand All @@ -167,7 +163,7 @@ def add_result(
try:
test_result = json.loads(test_result_string, strict=False)
except json.decoder.JSONDecodeError:
raise Exception(f"Could not parse JSON from '{test_result_string}'")
raise Exception(f"Could not parse JSON from '{test_result_string}'") from None

file_name = Path(test_result["test"])

Expand Down Expand Up @@ -229,10 +225,7 @@ def add_result(
)
current_test += 1
elif forward_stderr is not None and process_result.stderr.strip() != "":
forward_stderr(
"Process did not fail but still there is stderr output:\n"
+ process_result.stderr
)
forward_stderr("Process did not fail but still there is stderr output:\n" + process_result.stderr)

if on_progress_change is not None:
on_progress_change(
Expand Down Expand Up @@ -281,13 +274,9 @@ def __init__(
self.forward_stderr_function: Callable[[str], None] | None
if forward_stderr:
if self.silent:
self.forward_stderr_function = lambda message: print(
message, file=sys.stderr
)
self.forward_stderr_function = lambda message: print(message, file=sys.stderr)
else:
self.forward_stderr_function = lambda message: tqdm.write(
message, file=sys.stderr
)
self.forward_stderr_function = lambda message: tqdm.write(message, file=sys.stderr)
else:
self.forward_stderr_function = None

Expand All @@ -301,9 +290,7 @@ def find_tests(self, pattern: str, ignore: str) -> None:
if Path(pattern).resolve().is_file():
self.files = [Path(pattern).resolve()]
else:
ignored_files = set(
glob.iglob(str(self.test262_root / ignore), recursive=True)
)
ignored_files = set(glob.iglob(str(self.test262_root / ignore), recursive=True))
for path in glob.iglob(str(self.test262_root / pattern), recursive=True):
found_path = Path(path)
if (
Expand Down Expand Up @@ -340,7 +327,7 @@ def build_directory_result_map(self) -> None:
directory = file.relative_to(self.test262_root).parent
counter = self.directory_result_map
for segment in directory.parts:
if not segment in counter:
if segment not in counter:
counter[segment] = {"count": 1, "results": {}, "children": {}}
for result in TestResult:
counter[segment]["results"][result] = 0
Expand Down Expand Up @@ -375,9 +362,7 @@ def print_tree(tree, path, level):
passed = tree["results"][TestResult.PASSED]
percentage = (passed / count) * 100
pad = " " * (80 - len(path))
self.print_output(
f"{path}{pad}{passed:>5}/{count:<5} ({percentage:6.2f}%) {results} "
)
self.print_output(f"{path}{pad}{passed:>5}/{count:<5} ({percentage:6.2f}%) {results} ")
if passed > 0:
for k, v in tree["children"].items():
print_tree(v, path + "/" + k, level + 1)
Expand All @@ -400,13 +385,11 @@ def process_list(self, files: list[Path]) -> list[TestRun]:
on_progress_change=self.update_function,
forward_stderr=self.forward_stderr_function,
)
except Exception as e:
except Exception:
return [
TestRun(
file,
result=(
TestResult.RUNNER_EXCEPTION if i == 0 else TestResult.SKIPPED
),
result=(TestResult.RUNNER_EXCEPTION if i == 0 else TestResult.SKIPPED),
output=traceback.format_exc() if i == 0 else "",
exit_code=None,
strict_mode=None,
Expand All @@ -432,11 +415,10 @@ def run(self) -> None:
work_lists[index % amount_of_work_lists].append(test_path)

if not self.silent:
progressbar = tqdm(
total=self.total_count, mininterval=1, unit="tests", smoothing=0.1
)
progressbar = tqdm(total=self.total_count, mininterval=1, unit="tests", smoothing=0.1)
total_stats = Counter()

def update_progress(value, new_results, total_stats=Counter()):
def update_progress(value, new_results):
progress_mutex.acquire()
total_stats.update(new_results)
try:
Expand All @@ -455,18 +437,13 @@ def write_output(message: Any):
start = datetime.datetime.now()

with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
futures = [
executor.submit(self.process_list, file_list)
for file_list in work_lists
]
futures = [executor.submit(self.process_list, file_list) for file_list in work_lists]

for future in concurrent.futures.as_completed(futures):
test_runs = future.result()
for test_run in test_runs:
self.count_result(test_run)
if self.verbose or (
self.fail_only and test_run.result not in NON_FAIL_RESULTS
):
if self.verbose or (self.fail_only and test_run.result not in NON_FAIL_RESULTS):
self.print_output(
f"{EMOJIS[test_run.result]} {test_run.file}"
f"{' (strict mode)' if test_run.strict_mode else ''}"
Expand All @@ -480,9 +457,7 @@ def write_output(message: Any):
signalnum = test_run.exit_code * -1
if not test_run.output:
self.print_output("")
self.print_output(
f"{signal.strsignal(signalnum)}: {signalnum}"
)
self.print_output(f"{signal.strsignal(signalnum)}: {signalnum}")
self.print_output("")

if not self.silent:
Expand All @@ -498,9 +473,7 @@ def default_test262_runner_path() -> Path | None:
ladybird_source_dir = os.environ.get("LADYBIRD_SOURCE_DIR")

if ladybird_source_dir:
default_test262_runner = (
Path(ladybird_source_dir) / "Build" / "release" / "bin" / "test262-runner"
)
default_test262_runner = Path(ladybird_source_dir) / "Build" / "release" / "bin" / "test262-runner"

if default_test262_runner.exists():
return default_test262_runner
Expand Down Expand Up @@ -555,9 +528,7 @@ def main() -> None:
type=int,
help="memory limit for each test run in megabytes (defaults to 512)",
)
parser.add_argument(
"--json", action="store_true", help="print the test results as JSON"
)
parser.add_argument("--json", action="store_true", help="print the test results as JSON")
parser.add_argument(
"--per-file",
default=None,
Expand All @@ -572,13 +543,9 @@ def main() -> None:
action="store_true",
help="don't print any progress information",
)
logging_group.add_argument(
"-v", "--verbose", action="store_true", help="print output of test runs"
)
logging_group.add_argument("-v", "--verbose", action="store_true", help="print output of test runs")

parser.add_argument(
"-f", "--fail-only", action="store_true", help="only show failed tests"
)
parser.add_argument("-f", "--fail-only", action="store_true", help="only show failed tests")
parser.add_argument(
"--parse-only",
action="store_true",
Expand Down
38 changes: 13 additions & 25 deletions per_file_result_diff.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,25 @@
#!/usr/bin/env python3

# Copyright (c) 2021, Matthew Olsson <mattco@serenityos.org>
#
# SPDX-License-Identifier: MIT

import json

from argparse import ArgumentParser
from pathlib import Path
from main import TestResult, EMOJIS

from main import EMOJIS
from main import TestResult


class ResultParser:
def __init__(
self, old_path: Path, new_path: Path, regressions: bool, intersection_only: bool
) -> None:
def __init__(self, old_path: Path, new_path: Path, regressions: bool, intersection_only: bool) -> None:
old_results = json.loads(old_path.read_text())
new_results = json.loads(new_path.read_text())

self.duration_delta = float(new_results["duration"]) - float(
old_results["duration"]
)
self.old_results: dict[str, str] = {
k: v for k, v in sorted(old_results["results"].items())
}
self.duration_delta = float(new_results["duration"]) - float(old_results["duration"])
self.old_results: dict[str, str] = {k: v for k, v in sorted(old_results["results"].items())}
self.new_results: dict[str, str] = new_results["results"]
self.regressions = regressions
self.intersection_only = intersection_only
Expand Down Expand Up @@ -147,18 +145,14 @@ def print_full_results(self) -> None:
for path, result in self.diff_tests.items():
old_emoji = EMOJIS[result["old_result"]]
new_emoji = EMOJIS[result["new_result"]]
print(
f" {path:{self.longest_path_length}s} {old_emoji} -> {new_emoji}"
)
print(f" {path:{self.longest_path_length}s} {old_emoji} -> {new_emoji}")

def print_regressions(self) -> None:
for path, result in self.diff_tests.items():
if result["old_result"] == "PASSED":
old_emoji = EMOJIS[TestResult.PASSED]
new_emoji = EMOJIS[result["new_result"]]
print(
f" {path:{self.longest_path_length}s} {old_emoji} -> {new_emoji}"
)
print(f" {path:{self.longest_path_length}s} {old_emoji} -> {new_emoji}")

def print_results(self) -> None:
if self.regressions:
Expand All @@ -169,12 +163,8 @@ def print_results(self) -> None:

def main() -> None:
parser = ArgumentParser(description="Compare per-file test262 results")
parser.add_argument(
"-o", "--old", required=True, metavar="PATH", help="the path to the old results"
)
parser.add_argument(
"-n", "--new", required=True, metavar="PATH", help="the path to the new results"
)
parser.add_argument("-o", "--old", required=True, metavar="PATH", help="the path to the old results")
parser.add_argument("-n", "--new", required=True, metavar="PATH", help="the path to the new results")
parser.add_argument(
"-r",
"--regressions",
Expand All @@ -189,9 +179,7 @@ def main() -> None:
)
args = parser.parse_args()

ResultParser(
Path(args.old), Path(args.new), args.regressions, args.intersection_only
).print_results()
ResultParser(Path(args.old), Path(args.new), args.regressions, args.intersection_only).print_results()


if __name__ == "__main__":
Expand Down
13 changes: 13 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[tool.ruff]
line-length = 120

[tool.ruff.lint]
# https://docs.astral.sh/ruff/rules/
extend-select = [
"B", # flake8-bugbear
"I", # isort
]

[tool.ruff.lint.isort]
force-single-line = true
lines-between-types = 1
Loading