diff --git a/.gitignore b/.gitignore index a6f7de69..5235f25e 100644 --- a/.gitignore +++ b/.gitignore @@ -58,8 +58,8 @@ ENV/ # notebook checkpoints .ipynb_checkpoints/ -# isort -*.py.isorted +# ruff +.ruff_cache/ # OS generated files .DS_Store diff --git a/AGENTS.md b/AGENTS.md index af6f4bd7..a9099b45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,12 +102,10 @@ NB: `variables.mk` contains both runtime environment variables and variables tha ## Code style -* Formatting: black (line length 100) -* Import sorting: isort (black-compatible profile) -* Linting: flake8 -* Docstrings: pydocstyle -* Run `make lint` to apply formatting and check all of the above -* Configuration is in `pyproject.toml` and `setup.cfg` +* Formatting and linting: ruff (`ruff format` for formatting, `ruff check` for import sorting, lint + rules and docstrings), line length 100 +* Run `make lint` to apply formatting and auto-fixes, and check all of the above +* Configuration is in `pyproject.toml` * Functions should use type hints * New ForecastBench Python files should not add `from __future__ import annotations` * Prefer f-strings for Python string interpolation, including log messages. Use logger diff --git a/Makefile b/Makefile index 4613d9a5..9b35fb63 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,8 @@ export DEFAULT_CLOUD_FUNCTION_ENV_VARS=CLOUD_PROJECT=$(CLOUD_PROJECT),GOOGLE_CLO .PHONY: all clean lint test deploy install-requirements setup-python-env MAKE_LINT_ERROR_OUT ?= 0 -ISORT_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check-only,) -BLACK_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check,) +RUFF_CHECK_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),,--fix --exit-zero) +RUFF_FORMAT_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check,) VENV_PYTHON := .venv/bin/python ROOT_REQUIREMENTS_STAMP := .venv/.root-requirements-installed ALL_REQUIREMENTS_STAMP := .venv/.all-requirements-installed @@ -55,10 +55,9 @@ ROOT_REQUIREMENTS_NO_RUNTIME := .venv/.root-requirements-no-runtime.txt SRC_REQUIREMENTS := $(shell find src -type f -name requirements.txt ! -path '*/upload/*') lint: - isort $(ISORT_FLAGS) . - black $(BLACK_FLAGS) . - flake8 . - pydocstyle . + ruff check $(RUFF_CHECK_FLAGS) . + ruff format $(RUFF_FORMAT_FLAGS) . + ruff check . test: setup-python-env @. ${ROOT_DIR}.venv/bin/activate && python -m pytest src/tests/ $(ARGS) diff --git a/paper/category_table/main.py b/paper/category_table/main.py index 250b9cde..98d20af5 100644 --- a/paper/category_table/main.py +++ b/paper/category_table/main.py @@ -7,7 +7,7 @@ from variables import BUCKET_NAME sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from src.helpers import resolution # noqa: E402 +from src.helpers import resolution dfm = pd.read_json( f"gs://{BUCKET_NAME}/question_metadata.jsonl", diff --git a/paper/llm_breakdown_tables/main.py b/paper/llm_breakdown_tables/main.py index 560a06a2..f4caebf4 100644 --- a/paper/llm_breakdown_tables/main.py +++ b/paper/llm_breakdown_tables/main.py @@ -15,7 +15,7 @@ ) sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from src.helpers import resolution # noqa: E402 +from src.helpers import resolution def make_list_hashable(df, col): diff --git a/paper/source_table/main.py b/paper/source_table/main.py index 66bf4aef..ebaf4720 100644 --- a/paper/source_table/main.py +++ b/paper/source_table/main.py @@ -8,7 +8,7 @@ from variables import BUCKET_NAME sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from src.helpers import resolution # noqa: E402 +from src.helpers import resolution question_counts = {} df = pd.DataFrame() @@ -110,4 +110,4 @@ def get_source_values(df, source_name): \multicolumn{{2}}{{l}}{{\textbf{{Question Bank Total}}}} & {total_n} & {total_pairs} \\ \bottomrule \end{{tabular}} -""") # noqa: B950 +""") # noqa: E501 diff --git a/pyproject.toml b/pyproject.toml index 68b8e83a..4fa0a7e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,34 @@ testpaths = ["src/tests"] pythonpath = ["src"] -[tool.black] +[tool.ruff] line-length = 100 -force-exclude = ''' -( - ^/.venv.* - | ^/experiments/stability-analysis/ - | ^/experiments/ranking-simulation/ -) -''' +target-version = "py314" +src = [".", "src"] +extend-exclude = [ + ".venv*", + "experiments/stability-analysis", + "experiments/ranking-simulation", + "upload*", +] + +[tool.ruff.lint] +# Parity with the previous isort + flake8 (+ bugbear B950) + pydocstyle setup. +select = ["E", "W", "F", "B", "I", "D"] +# E203/E701 are formatter-owned; B9xx is bugbear's opt-in set (B950 is emulated by E501 below). +ignore = ["E203", "E701", "B9"] + +[tool.ruff.lint.pycodestyle] +# E501 at 110 reproduces bugbear's B950 (line-length + 10 %). +max-line-length = 110 + +[tool.ruff.lint.pydocstyle] +convention = "pep257" + +[tool.ruff.lint.per-file-ignores] +"**/tests/**" = ["D"] +"src/sources/fred_questions.py" = ["E501"] +"src/sources/wikipedia.py" = ["E501"] + +[tool.ruff.format] +exclude = ["*.md"] diff --git a/requirements.txt b/requirements.txt index 661f8d26..a1c1a0d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,4 @@ -r requirements.runtime.txt -isort==6.0.0 -black[jupyter]==26.3.1 -flake8==7.1.2 -flake8-bugbear==24.12.12 -pydocstyle==6.3.0 +ruff==0.17.1 pytest==9.0.3 pytest-mock==3.15.1 diff --git a/scripts/generate_yfinance_pairs.py b/scripts/generate_yfinance_pairs.py index 93a50eb3..b3d402d3 100644 --- a/scripts/generate_yfinance_pairs.py +++ b/scripts/generate_yfinance_pairs.py @@ -171,8 +171,7 @@ def main() -> None: "--seed", type=int, default=DEFAULT_SEED, - help="Seed for drawing new pairs; existing pairs are never redrawn " - "(default: %(default)s).", + help="Seed for drawing new pairs; existing pairs are never redrawn (default: %(default)s).", ) args = parser.parse_args() if not args.question_file.is_file(): diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index d27b94de..00000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[isort] -skip_glob = .venv* -skip = stability-analysis, - ranking-simulation -profile = black - -[flake8] -# some rules from https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8 -exclude = - .venv*, - stability-analysis, - ranking-simulation, -max-line-length = 100 -extend-select = B950 -extend-ignore = E203,E501,E701 - -[pydocstyle] -match = .*\.py -match-dir = ^(?!(\.venv|stability-analysis|ranking-simulation|tests)).*$ diff --git a/src/base_eval/experiments/llm_crowd_candidate_testing.ipynb b/src/base_eval/experiments/llm_crowd_candidate_testing.ipynb index b271d28a..14969a41 100644 --- a/src/base_eval/experiments/llm_crowd_candidate_testing.ipynb +++ b/src/base_eval/experiments/llm_crowd_candidate_testing.ipynb @@ -157,7 +157,6 @@ "def executor(max_workers, prompt, model_name, save_dict):\n", " \"\"\"Scratchpad executor.\"\"\"\n", " with ThreadPoolExecutor(max_workers=max_workers) as executor:\n", - "\n", " worker_with_args = partial(\n", " worker, prompt=prompt, model_name=model_name, save_dict=save_dict\n", " )\n", @@ -207,9 +206,9 @@ " results[model] = {}\n", " try:\n", " results[model] = read_jsonl(file_path)\n", - " model_result_loaded[model][\n", - " f\"prompt_{prompt_index}\"\n", - " ] = True # Set flag to True if loaded successfully\n", + " model_result_loaded[model][f\"prompt_{prompt_index}\"] = (\n", + " True # Set flag to True if loaded successfully\n", + " )\n", " except:\n", " results[model][f\"prompt_{prompt_index}\"] = {i: \"\" for i in range(len(questions_list))}\n", "\n", diff --git a/src/base_eval/experiments/scratchpad_testing.ipynb b/src/base_eval/experiments/scratchpad_testing.ipynb index 6dc0280f..3f5ee620 100644 --- a/src/base_eval/experiments/scratchpad_testing.ipynb +++ b/src/base_eval/experiments/scratchpad_testing.ipynb @@ -411,7 +411,6 @@ "def executor(max_workers, prompt, model_name, save_dict):\n", " \"\"\"Scratchpad executor.\"\"\"\n", " with ThreadPoolExecutor(max_workers=max_workers) as executor:\n", - "\n", " worker_with_args = partial(\n", " worker, prompt=prompt, model_name=model_name, save_dict=save_dict\n", " )\n", @@ -445,9 +444,9 @@ " results[model] = {}\n", " try:\n", " results[model] = read_jsonl(file_path)\n", - " model_result_loaded[model][\n", - " f\"prompt_{prompt_index}\"\n", - " ] = True # Set flag to True if loaded successfully\n", + " model_result_loaded[model][f\"prompt_{prompt_index}\"] = (\n", + " True # Set flag to True if loaded successfully\n", + " )\n", " except:\n", " results[model][f\"prompt_{prompt_index}\"] = {i: \"\" for i in range(len(questions_list))}\n", "\n", diff --git a/src/base_eval/naive_and_dummy_forecasters/main.py b/src/base_eval/naive_and_dummy_forecasters/main.py index 9d165df8..0d78f63a 100644 --- a/src/base_eval/naive_and_dummy_forecasters/main.py +++ b/src/base_eval/naive_and_dummy_forecasters/main.py @@ -14,7 +14,7 @@ from utils import gcp sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import ( # noqa: E402 +from helpers import ( acled, constants, data_utils, @@ -24,7 +24,7 @@ wikipedia, yfinance, ) -from orchestration import _io # noqa: E402 +from orchestration import _io logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -392,7 +392,7 @@ def prepare_df_and_set_null_values(df, forecast_due_date, last_date_for_data): # Expand resolution dates df["resolution_dates"] = df.apply( - lambda x: ([] if x["source"] in resolution.MARKET_SOURCES else x["resolution_dates"]), + lambda x: [] if x["source"] in resolution.MARKET_SOURCES else x["resolution_dates"], axis=1, ) df = df.explode("resolution_dates", ignore_index=True) diff --git a/src/helpers/dbnomics.py b/src/helpers/dbnomics.py index 2780c74e..66cf5709 100644 --- a/src/helpers/dbnomics.py +++ b/src/helpers/dbnomics.py @@ -5,7 +5,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -from sources._metadata import SOURCE_METADATA # noqa: E402 +from sources._metadata import SOURCE_METADATA _META = SOURCE_METADATA["dbnomics"] SOURCE_INTRO = _META["source_intro"] diff --git a/src/helpers/fred.py b/src/helpers/fred.py index 79fed762..b62132c4 100644 --- a/src/helpers/fred.py +++ b/src/helpers/fred.py @@ -10,7 +10,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -from sources._metadata import SOURCE_METADATA # noqa: E402 +from sources._metadata import SOURCE_METADATA _META = SOURCE_METADATA["fred"] SOURCE_INTRO = _META["source_intro"] diff --git a/src/helpers/metadata_prompts.py b/src/helpers/metadata_prompts.py index b9c60db4..c214df51 100644 --- a/src/helpers/metadata_prompts.py +++ b/src/helpers/metadata_prompts.py @@ -66,4 +66,4 @@ Insert thinking: {{ insert your concise thoughts here }} Classification: -{{ insert "flag" or "ok"}}""" # noqa: B950 +{{ insert "flag" or "ok"}}""" # noqa: E501 diff --git a/src/helpers/polymarket.py b/src/helpers/polymarket.py index f35aea64..3d52bae7 100644 --- a/src/helpers/polymarket.py +++ b/src/helpers/polymarket.py @@ -5,7 +5,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -from sources._metadata import SOURCE_METADATA # noqa: E402 +from sources._metadata import SOURCE_METADATA _META = SOURCE_METADATA["polymarket"] SOURCE_INTRO = _META["source_intro"] diff --git a/src/helpers/question_curation.py b/src/helpers/question_curation.py index a17deaae..5d1f7a07 100644 --- a/src/helpers/question_curation.py +++ b/src/helpers/question_curation.py @@ -28,9 +28,9 @@ FREEZE_QUESTION_SOURCES = {**FREEZE_QUESTION_MARKET_SOURCES, **FREEZE_QUESTION_DATA_SOURCES} for _sources in (FREEZE_QUESTION_MARKET_SOURCES, FREEZE_QUESTION_DATA_SOURCES): - assert math.isclose( - sum(source["sampling_share"] for source in _sources.values()), 1 - ), "sampling shares within a source type must sum to 1" + assert math.isclose(sum(source["sampling_share"] for source in _sources.values()), 1), ( + "sampling shares within a source type must sum to 1" + ) DATA_SOURCES = list(FREEZE_QUESTION_DATA_SOURCES.keys()) MARKET_SOURCES = list(FREEZE_QUESTION_MARKET_SOURCES.keys()) diff --git a/src/helpers/slack.py b/src/helpers/slack.py index 9e8203d7..9b1b7632 100644 --- a/src/helpers/slack.py +++ b/src/helpers/slack.py @@ -5,7 +5,7 @@ from slack_sdk import WebClient from slack_sdk.errors import SlackApiError -from . import keys # noqa: E402 +from . import keys logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/helpers/wikipedia.py b/src/helpers/wikipedia.py index 10a9bcbd..01cebff9 100644 --- a/src/helpers/wikipedia.py +++ b/src/helpers/wikipedia.py @@ -23,11 +23,11 @@ # Private imports from the source module are deliberate: this helper only exists until # `base_eval` is refactored to call `WikipediaSource.get_naive_forecast()`, after which it is # deleted and nothing outside `sources.wikipedia` needs these. -from sources.wikipedia import _PAGES as PAGES # noqa: E402 -from sources.wikipedia import ( # noqa: E402 +from sources.wikipedia import _PAGES as PAGES +from sources.wikipedia import ( _WIKIPEDIA_QUESTION_BANK_DATA_STORAGE_START_DATE as WIKIPEDIA_QUESTION_BANK_DATA_STORAGE_START_DATE, ) -from sources.wikipedia import QuestionType # noqa: E402 +from sources.wikipedia import QuestionType logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/leaderboard/main.py b/src/leaderboard/main.py index 4eea6a33..5bfc560f 100644 --- a/src/leaderboard/main.py +++ b/src/leaderboard/main.py @@ -23,9 +23,9 @@ sys.path.append(os.path.dirname(__file__)) sys.path.append(os.path.join(os.path.dirname(__file__), "..")) -import llm_identities # noqa: E402 +import llm_identities -from helpers import ( # noqa: E402 +from helpers import ( constants, data_utils, dates, @@ -34,11 +34,11 @@ resolution, slack, ) -from llm_forecaster.forecast_variants import ( # noqa: E402 +from llm_forecaster.forecast_variants import ( ALL_FORECAST_VARIANT_KEYS_WITH_CONTEXT, ALL_FORECAST_VARIANT_KEYS_WITHOUT_CONTEXT, ) -from sources import DATASET_SOURCE_NAMES, MARKET_SOURCE_NAMES # noqa: E402 +from sources import DATASET_SOURCE_NAMES, MARKET_SOURCE_NAMES logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -300,11 +300,10 @@ def get_org_logo(org: str) -> str: f"Percentage of {N_REPLICATES:,} simulations in which this model ranked in the top 5%." ), "Peer": ( - "Peer score relative to the average Brier score on each question. " - "Higher scores are better." + "Peer score relative to the average Brier score on each question. Higher scores are better." ), "BSS": ( - "Brier Skill Score using the ForecastBench Naive Forecaster. " "Higher scores are better." + "Brier Skill Score using the ForecastBench Naive Forecaster. Higher scores are better." ), } @@ -787,8 +786,7 @@ def write_question_fixed_effects( # One file per leaderboard, overwritten nightly. The history lives in the dataset repo, # where `push_datasets_to_git` sends this file every night. local_filename = ( - f"{directory}/question-fixed-effects/" - f"question_fixed_effects.{leaderboard_file_stem}.json" + f"{directory}/question-fixed-effects/question_fixed_effects.{leaderboard_file_stem}.json" ) os.makedirs(os.path.dirname(local_filename), exist_ok=True) df.to_json(local_filename, orient="records") @@ -2020,10 +2018,10 @@ def format_ci(df, question_type): col_name = get_comparison_p_val_col(comparison) col_name_simple = col_name + "_simple" df[col_name_simple] = df[col_name].apply( - lambda p: ("Yes" if p < 0.01 else "Likely" if p < 0.1 else "No") + lambda p: "Yes" if p < 0.01 else "Likely" if p < 0.1 else "No" ) df[col_name] = df[col_name].apply( - lambda p: ("<0.001" if p < 0.001 else "<0.01" if p < 0.01 else f"{p:.2f}") + lambda p: "<0.001" if p < 0.001 else "<0.01" if p < 0.01 else f"{p:.2f}" ) # Set the p-value for the best to N/A comparison_idx = get_comparison_model_index(df=df, comparison=comparison) @@ -2190,10 +2188,10 @@ def format_ci(df, question_type): col_name = get_comparison_p_val_col(comparison) col_name_simple = col_name + "_simple" df[col_name_simple] = df[col_name].apply( - lambda p: ("Yes" if p < 0.01 else "Likely" if p < 0.1 else "No") + lambda p: "Yes" if p < 0.01 else "Likely" if p < 0.1 else "No" ) df[col_name] = df[col_name].apply( - lambda p: ("<0.001" if p < 0.001 else "<0.01" if p < 0.01 else f"{p:.2f}") + lambda p: "<0.001" if p < 0.001 else "<0.01" if p < 0.01 else f"{p:.2f}" ) # Set the p-value for the best to N/A comparison_idx = get_comparison_model_index(df=df, comparison=comparison) @@ -2467,9 +2465,9 @@ def brier_skill_score(df: pd.DataFrame) -> pd.DataFrame: ref_brier_by_question = ref_data.set_index("question_pk")["brier_score"].to_dict() # Ensure the naive forecaster has forecast on all questions across all question sets - assert ( - df["question_pk"].isin(ref_brier_by_question).all() - ), "Reference model must predict all questions across all question sets" + assert df["question_pk"].isin(ref_brier_by_question).all(), ( + "Reference model must predict all questions across all question sets" + ) # Calculate Brier skill score per question df["ref_brier"] = df["question_pk"].map(ref_brier_by_question) @@ -2679,7 +2677,7 @@ def generate_simulated_leaderboards( seed = env.RANDOM_SEED def bootstrap_and_score(idx): - logger.info(f"[replicate {idx+1}/{N}] starting...") + logger.info(f"[replicate {idx + 1}/{N}] starting...") random_state = None if seed is None else np.random.RandomState(seed + idx) df_bs = ( df.groupby(["forecast_due_date", "source"]) diff --git a/src/llm_forecaster/prompts.py b/src/llm_forecaster/prompts.py index ef8bd381..f014ce61 100644 --- a/src/llm_forecaster/prompts.py +++ b/src/llm_forecaster/prompts.py @@ -60,7 +60,7 @@ def render_template(template: str, params: Mapping[str, Any]) -> str: Output your answer (a number between 0 and 1) with an asterisk at the beginning and end of the decimal. Do not output anything else. Answer: {{ Insert answer here }} -""" # noqa: B950 +""" # noqa: E501 ZERO_SHOT_MARKET_WITH_FREEZE_VALUE_PROMPT = """ You are an expert superforecaster, familiar with the work of Tetlock and others. Make a prediction of the probability that the question will be resolved as true. You MUST give a probability estimate between 0 and 1 UNDER ALL CIRCUMSTANCES. If for some reason you can't answer, pick the base rate, but return a number between 0 and 1. @@ -84,7 +84,7 @@ def render_template(template: str, params: Mapping[str, Any]) -> str: Output your answer (a number between 0 and 1) with an asterisk at the beginning and end of the decimal. Do not output anything else. Answer: {{ Insert answer here }} -""" # noqa: B950 +""" # noqa: E501 ZERO_SHOT_DATASET_PROMPT = """ @@ -113,7 +113,7 @@ def render_template(template: str, params: Mapping[str, Any]) -> str: Output your answer (a number between 0 and 1) with an asterisk at the beginning and end of the decimal. (For example, if there are n resolution dates, you would output different *p* for each resolution date) Do not output anything else. Answer: {{ Insert answer here }} -""" # noqa: B950 +""" # noqa: E501 FORECAST_EXTRACTION_PROMPT = """ @@ -140,4 +140,4 @@ def render_template(template: str, params: Mapping[str, Any]) -> str: ```text {model_response} ``` -""" # noqa: B950 +""" # noqa: E501 diff --git a/src/metadata/tag_questions/main.py b/src/metadata/tag_questions/main.py index d1fb2b98..e87993c2 100644 --- a/src/metadata/tag_questions/main.py +++ b/src/metadata/tag_questions/main.py @@ -15,7 +15,7 @@ from utils import gcp sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import ( # noqa: E402 +from helpers import ( constants, data_utils, decorator, diff --git a/src/metadata/validate_questions/main.py b/src/metadata/validate_questions/main.py index ce7b52b9..4fe34a8d 100644 --- a/src/metadata/validate_questions/main.py +++ b/src/metadata/validate_questions/main.py @@ -11,7 +11,7 @@ from utils import gcp sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import ( # noqa: E402 +from helpers import ( constants, data_utils, decorator, diff --git a/src/nightly_update_workflow/compress_buckets/main.py b/src/nightly_update_workflow/compress_buckets/main.py index 013f2ce5..35630de6 100644 --- a/src/nightly_update_workflow/compress_buckets/main.py +++ b/src/nightly_update_workflow/compress_buckets/main.py @@ -10,7 +10,7 @@ from termcolor import colored sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import decorator, env # noqa: E402 +from helpers import decorator, env logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/nightly_update_workflow/manager/main.py b/src/nightly_update_workflow/manager/main.py index 89036f23..86d4b919 100644 --- a/src/nightly_update_workflow/manager/main.py +++ b/src/nightly_update_workflow/manager/main.py @@ -8,8 +8,8 @@ from tabulate import tabulate sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import cloud_run, constants, env, question_curation, slack # noqa: E402 -from sources import ALL_SOURCE_NAMES # noqa: E402 +from helpers import cloud_run, constants, env, question_curation, slack +from sources import ALL_SOURCE_NAMES logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/src/nightly_update_workflow/worker/main.py b/src/nightly_update_workflow/worker/main.py index 7b07aea2..ae6128f1 100644 --- a/src/nightly_update_workflow/worker/main.py +++ b/src/nightly_update_workflow/worker/main.py @@ -4,9 +4,9 @@ import sys sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) -from helpers import cloud_run, dates, env, question_curation # noqa: E402 -from orchestration import _io # noqa: E402 -from sources import ALL_SOURCE_NAMES, SOURCE_METADATA # noqa: E402 +from helpers import cloud_run, dates, env, question_curation +from orchestration import _io +from sources import ALL_SOURCE_NAMES, SOURCE_METADATA metadata = [ [ diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index 8ba1a67b..ce6c0bd6 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -267,7 +267,7 @@ def push_datasets_to_git() -> None: end of the run rather than having each of the parallel resolve and leaderboard jobs push its own files, which avoids the race condition of concurrent pushes to the repository. """ - from helpers import git, keys # noqa: E402 + from helpers import git, keys if env.RUNNING_LOCALLY: logger.info("Running locally; not pushing datasets to git.") diff --git a/src/orchestration/func_llm_forecaster_worker/main.py b/src/orchestration/func_llm_forecaster_worker/main.py index 3e268a40..2c161151 100644 --- a/src/orchestration/func_llm_forecaster_worker/main.py +++ b/src/orchestration/func_llm_forecaster_worker/main.py @@ -166,8 +166,7 @@ def _log_written_forecast_files( _written_forecast_file_summary(written_file) for written_file in written_files ] logger.info( - f"Wrote and uploaded {len(written_files)} LLM forecast files: " - f"{'; '.join(file_summaries)}" + f"Wrote and uploaded {len(written_files)} LLM forecast files: {'; '.join(file_summaries)}" ) diff --git a/src/orchestration/func_resolve/main.py b/src/orchestration/func_resolve/main.py index 1f8e870a..bb400528 100644 --- a/src/orchestration/func_resolve/main.py +++ b/src/orchestration/func_resolve/main.py @@ -130,8 +130,7 @@ def get_df_len(df, single, sources): "." if n_combo == 0 else f", No. Combo Q's {n_combo:,}/{n_orig:,} Orig No. Q's." ) logger.info( - f" * {source} No. Single Q's: {n_single:,}/{n_orig:,} " - f"Orig No. Q's{combo_info}" + f" * {source} No. Single Q's: {n_single:,}/{n_orig:,} Orig No. Q's{combo_info}" ) n_single = get_df_len(df=df, single=True, sources=sources) n_combo = get_df_len(df=df, single=False, sources=sources) diff --git a/src/question_sampling/_binning.py b/src/question_sampling/_binning.py index 29535bd1..aecbeab2 100644 --- a/src/question_sampling/_binning.py +++ b/src/question_sampling/_binning.py @@ -78,7 +78,7 @@ def get_market_value_bin(value: float | str | None) -> str: try: val = float(value) - except (ValueError, TypeError): + except ValueError, TypeError: return "unknown" for bin_config in MARKET_VALUE_CONFIG: @@ -115,7 +115,7 @@ def get_time_horizon_bin(close_datetime: str | None, forecast_datetime: datetime if horizon_config["min"] <= days_until_close <= horizon_config["max"]: return get_bin_label(horizon_config, "time_horizon") return "unknown" - except (ValueError, TypeError): + except ValueError, TypeError: return "unknown" diff --git a/src/resolve/_prepare.py b/src/resolve/_prepare.py index c4e1b1a2..b2df0810 100644 --- a/src/resolve/_prepare.py +++ b/src/resolve/_prepare.py @@ -28,7 +28,7 @@ def convert_and_bound_dates(date_str): """Safely convert dates to datetime, setting max date for dates that are too large.""" try: return pd.to_datetime(date_str) - except (OutOfBoundsDatetime, OverflowError): + except OutOfBoundsDatetime, OverflowError: return pd.Timestamp("2262-04-11") diff --git a/src/sources/_market.py b/src/sources/_market.py index 86610b70..f9be04ef 100644 --- a/src/sources/_market.py +++ b/src/sources/_market.py @@ -183,7 +183,7 @@ def _get_market_resolution_date(row): def to_date_or_max(s): try: return dates.convert_zulu_to_datetime(s).date() - except (ValueError, TypeError): + except ValueError, TypeError: return date.max close_date = to_date_or_max(row["market_info_close_datetime"].iloc[0]) diff --git a/src/sources/_metadata.py b/src/sources/_metadata.py index e4d0dddc..fcb5ac4f 100644 --- a/src/sources/_metadata.py +++ b/src/sources/_metadata.py @@ -812,204 +812,204 @@ NullifiedQuestion( id="149b5a465d9640ee10afcd1c6dde90627a4b58918111c14455d369f304aae454", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="98e72a2d4c6daa0b0d8aee1d02a8628bbacf713f0e44b02f80a12b1dae1c618f", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), # Name changed: "Erigaisi Arjun" --> "Arjun Erigaisi" NullifiedQuestion( id="b70970a0440d1b7dedde9220fb60ffe3f2ed8b00ef12b45341772046caa12092", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), # Rameshbabu Praggnanandhaa — too many repeated name changes # NB: _not_ nullifying ff153a13... (first asked 2025-05-25) or a987eef3... (2025-03-30) NullifiedQuestion( id="7687186d5e0807f8925a694beafb3d6e057978a9a01f0d1a3e0eaf1a49959e78", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="479a40c45087510f72ee43a77aaccf78d563361728151ed3aab9b2b186db0b72", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="4b9175c88f855ee0d0fc54640158fc7da10b7b2dcc4fe1053bd180ac1a72bf39", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), # Virus common name changed from "Monkeypox" to "Mpox" NullifiedQuestion( id="f9323386a651ce67fc0da31285bee22a4ec53b8a2ea5220431ecb4560fb44c77", nullification_start_date=date(2022, 8, 21), - ), # noqa: B950 + ), NullifiedQuestion( id="3f04d0cfccd38b26e86c0939516c483eb31edf6aaa3a1eaaabe38a48f7a0996a", nullification_start_date=date(2022, 8, 21), - ), # noqa: B950 + ), # Leinier Domínguez Pérez — too many repeated name changes NullifiedQuestion( id="c8cc0816ce50a7fc018eccb7e6ed19628dc1f56e1cda26aca4b8f09c4edc7beb", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="21f7534aaa7292ba1e71ed0d1ce0fc350febe64414083b4b60d35765781eab35", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="9ab6734c6bf88f28a8c71b9d73995541b351f2663a7d8331a2c56dd5116d78a3", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="a9783d8184c3f43668cc21417788be00fd4ff70eec91064c5539ed5ebb0019e8", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="fa118e263e1218af8bb24cf7f6dd1c68e179d430584adf5b9b37d1b8488932d8", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="60d86f26a5b1e6576d218076ae7a66bf0fadc0bfe042ff1adf875918cc8d2781", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="6f8a3d10d39d69ecbdb10db2fabb66d852af39b95ce1af9f48ce5d9fd0175d87", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), NullifiedQuestion( id="dfa2dc6d7511437365132459a03e4d7bc10632ffd78c145fb98496699647f968", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), # Resolved keys from _TRANSFORM_ID_MAPPING — old erroneous IDs superseded # Tatjana Schoenmaker, lost swimming WR NullifiedQuestion( id="25891a351e97154028edc8075558470a6ec21d6d37dbd75f74268ee1b48253bf", nullification_start_date=date(2023, 7, 5), - ), # noqa: B950 + ), NullifiedQuestion( id="94297b75a6d18445c35a179a860b810bf0be7b6f296c502cec7caab24c8c1775", nullification_start_date=date(2023, 7, 5), - ), # noqa: B950 + ), # Anthony Ervin, lost swimming WR NullifiedQuestion( id="cf02d516cc8b14b7b2880baae0ca4d520b167fe271123e6adfeedaefb83a3ec5", nullification_start_date=date(2023, 8, 8), - ), # noqa: B950 + ), NullifiedQuestion( id="6358ab9dab0aa4b6fc2abe8aacf1b31c8cbed08d54557eb4982c230fe19fe774", nullification_start_date=date(2023, 8, 8), - ), # noqa: B950 + ), # Michael Phelps, lost swimming WR NullifiedQuestion( id="eea4cb0741c001c18ec28a58f64fb02bfba72e776f2d9ef2257309269b119526", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), NullifiedQuestion( id="234175128275d109b5ffe5f8a30f863f150051e892e56566f88936b961be1f2f", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), # Benedetta Pilato, lost swimming WR NullifiedQuestion( id="e4afa18eb3d8d08fbc37c114f876a93ddceac453da415512ef5d73c7d26f391d", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), NullifiedQuestion( id="747aa3406023deab8175b051bac64b55c061d38c2aebc73c1ded759de7b0477a", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), # Zac Stubblety-Cook, lost swimming WR NullifiedQuestion( id="5b078ec5a0d0a51c3668c62fe93441bd177ad4c58a1ff1d50b62a8bf6bc609fe", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), NullifiedQuestion( id="afd040f28eb27f973ba1dc2cfeb3f613a7c29a543b14cbab4ba8d44ca8eb0d36", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), # Federica Pellegrini, lost swimming WR NullifiedQuestion( id="6e295dc29db5dce0672097160d432e7a3af469317298cb3153d745b2270041f1", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), NullifiedQuestion( id="f0054684e6c6c24c5595e5cdf8498ffc5479e82d26a8b0318af35a26cd9b9ce7", nullification_start_date=date(2023, 8, 29), - ), # noqa: B950 + ), # Liu Xiang, lost swimming WR NullifiedQuestion( id="245eb0146484bad467bbdb3d0c871f30390fb1a902105f86c85ec4637c52a9f4", nullification_start_date=date(2023, 10, 20), - ), # noqa: B950 + ), NullifiedQuestion( id="e222aa0998ad2e53a4cbfbdb11f3d80dfd13a263b4748e4a6cd8f4b965f0506f", nullification_start_date=date(2023, 10, 20), - ), # noqa: B950 + ), # Hunter Armstrong, lost swimming WR NullifiedQuestion( id="851337578d0bf07dc60b233f5ef2a49d0309c1728621dd7b4ac0724414887fde", nullification_start_date=date(2023, 11, 13), - ), # noqa: B950 + ), NullifiedQuestion( id="56e00c66d9d2bfa3dd3ad0656c81701e04033438f90320ba96a63b62e61a4ea5", nullification_start_date=date(2023, 11, 13), - ), # noqa: B950 + ), # David Popovici, lost swimming WR NullifiedQuestion( id="646cd3619a16c273007816e559834682e19754dcaf7d0ecb6ffebe64d351f177", nullification_start_date=date(2024, 3, 21), - ), # noqa: B950 + ), NullifiedQuestion( id="0e0f5a6cf1ac926657d43b909af4d2fb27ba975dfe3a274fbe0930dcf667d499", nullification_start_date=date(2024, 3, 21), - ), # noqa: B950 + ), # Mollie O'Callaghan, lost swimming WR NullifiedQuestion( id="ebb4e1e85bed81266e94dda8e84eafe1479d5697f850792d84b5fab7251f483f", nullification_start_date=date(2024, 7, 18), - ), # noqa: B950 + ), NullifiedQuestion( id="b4c4989ac25edfbb8510e8ffa9aeee70c0de0d82e22a360faac590304f67c575", nullification_start_date=date(2024, 7, 18), - ), # noqa: B950 + ), # Sun Yang, lost swimming WR NullifiedQuestion( id="7558c5b4f539cc922552c4f18a9a5cdaccbc100d6108acf117e886bd9dc67857", nullification_start_date=date(2024, 8, 4), - ), # noqa: B950 + ), NullifiedQuestion( id="04bfcc27745a1813367fcb5aad43423db616dccff54c1cc929bd32de3f43a38a", nullification_start_date=date(2024, 8, 4), - ), # noqa: B950 + ), # Kate Douglass, lost swimming WR NullifiedQuestion( id="eaf10e98fdc5ddd2227b212f1e446a1937a2e0529b8f89c9a2528cb469e7cc27", nullification_start_date=date(2024, 11, 2), - ), # noqa: B950 + ), NullifiedQuestion( id="c539c3ef6d2534204b4fc67a94b14eebc7c51f141fea3c30f337cb3ede390b11", nullification_start_date=date(2024, 11, 2), - ), # noqa: B950 + ), # Katinka Hosszú, lost swimming WR NullifiedQuestion( id="2e88b046538e239140043da9471c2b4894615a12173c3a52ee707321acf2ed8d", nullification_start_date=date(2025, 6, 10), - ), # noqa: B950 + ), NullifiedQuestion( id="c4db6cf85ef3ef4165705b863f1491f2903df3a2534e2d4e25f57edcbdfaac4b", nullification_start_date=date(2025, 6, 10), - ), # noqa: B950 + ), # Vaccine was created in 2023 but Wikipedia table had not been updated NullifiedQuestion( id="242926fea271734ef8d4920e532414b38dbfdf301516fd9f0c988abd0ce777dd", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), # Ryan Lochte, lost swimming WR NullifiedQuestion( id="12486c21df689124f8fdad70760247dffe2b7696599748bcb5c7a738735285d5", nullification_start_date=date(2025, 7, 30), - ), # noqa: B950 + ), NullifiedQuestion( id="c6ee39b4504603aa5ddbe73f378d48d94ab128406e5dd1bbb70ead0207a43840", nullification_start_date=date(2025, 7, 30), - ), # noqa: B950 + ), # Non-existent swimmer "Jonny Liette" — fabricated 50m backstroke WR question. The name # is not a real record holder (the men's 50m backstroke WR is held by Kliment # Kolesnikov, 23.55) and never appeared in the source article's revision history. @@ -1017,7 +1017,7 @@ NullifiedQuestion( id="135a1523e67a012ce700a810c8c0c152a5bc6f0f01303ffe7ae28daf9ba4aa7c", nullification_start_date=BENCHMARK_START_DATE_DATETIME_DATE, - ), # noqa: B950 + ), ], }, "yfinance": { diff --git a/src/sources/dbnomics_questions.py b/src/sources/dbnomics_questions.py index 82ae4f82..9b75b70f 100644 --- a/src/sources/dbnomics_questions.py +++ b/src/sources/dbnomics_questions.py @@ -3,8 +3,6 @@ Only consumed by ``sources.dbnomics``; kept in a separate file purely for maintainability. """ -# flake8: noqa: B950 - METEOFRANCE_STATIONS = [ {"id": "07005", "station": "Abbeville"}, {"id": "07015", "station": "Lille Airport"}, diff --git a/src/sources/fred_questions.py b/src/sources/fred_questions.py index bf0667d7..5ada762a 100644 --- a/src/sources/fred_questions.py +++ b/src/sources/fred_questions.py @@ -4,8 +4,6 @@ purely for maintainability. """ -# flake8: noqa: B950 - FRED_QUESTIONS = [ { "id": "AAA10Y", diff --git a/src/sources/kalshi.py b/src/sources/kalshi.py index 75afc22c..4e99c7fb 100644 --- a/src/sources/kalshi.py +++ b/src/sources/kalshi.py @@ -857,7 +857,7 @@ def _resolution_window(market: dict) -> tuple[str, str] | None: name: dates.convert_zulu_to_datetime(timestamp) for name, timestamp in timestamps.items() } - except (TypeError, ValueError): + except TypeError, ValueError: return None if any( parsed_datetime.utcoffset() is None for parsed_datetime in parsed_datetimes.values() @@ -912,7 +912,7 @@ def _question_text(market: dict) -> str: yes_sub_title = (market.get("yes_sub_title") or "").strip() if not yes_sub_title or yes_sub_title.lower() == "yes": return market["title"] - return f'{market["title"]} [Yes: {yes_sub_title}]' + return f"{market['title']} [Yes: {yes_sub_title}]" @staticmethod def _market_url(series_ticker: str, event_ticker: str) -> str: diff --git a/src/sources/polymarket.py b/src/sources/polymarket.py index 62fba75c..f01641d2 100644 --- a/src/sources/polymarket.py +++ b/src/sources/polymarket.py @@ -56,8 +56,7 @@ def __init__(self, condition_ids): """Initialize the error with the complete failed condition ID list.""" self.condition_ids = list(condition_ids) super().__init__( - "Problem getting markets for condition ids: " - f"{json.dumps(self.condition_ids, indent=2)}" + f"Problem getting markets for condition ids: {json.dumps(self.condition_ids, indent=2)}" ) @@ -444,7 +443,7 @@ def get_resolved_ids_with_suspect_value(dfq: DataFrame[QuestionFrame] | None) -> ): try: is_resolution_value = float(freeze_value) in _VALID_RESOLUTION_VALUES - except (TypeError, ValueError): + except TypeError, ValueError: is_resolution_value = False if not is_resolution_value: suspect_ids.append(str(question_id)) diff --git a/src/sources/serpapi.py b/src/sources/serpapi.py index f23d4276..c8e8e9d0 100644 --- a/src/sources/serpapi.py +++ b/src/sources/serpapi.py @@ -287,8 +287,10 @@ def update( (requests.Timeout, requests.ConnectionError, requests.HTTPError), max_tries=3, jitter=None, - giveup=lambda exc: isinstance(exc, requests.HTTPError) - and (exc.response is None or exc.response.status_code not in {429, 500, 502, 503, 504}), + giveup=lambda exc: ( + isinstance(exc, requests.HTTPError) + and (exc.response is None or exc.response.status_code not in {429, 500, 502, 503, 504}) + ), logger=None, # Request exception URLs can contain the API key. ) def _get_response(self, params: dict, *, html_url: str | None = None) -> requests.Response: diff --git a/src/sources/wikipedia.py b/src/sources/wikipedia.py index 7efa0c6d..0855cc14 100644 --- a/src/sources/wikipedia.py +++ b/src/sources/wikipedia.py @@ -951,7 +951,6 @@ def _id_unhash(self, hash_key: str): return self.hash_mapping.get(hash_key) -# flake8: noqa: B950 _TRANSFORM_ID_MAPPING = { # Below is a list of IDs that have changed since question sets were released for reasons # explained below. diff --git a/src/sources/yfinance.py b/src/sources/yfinance.py index 0f1464bb..f738a5d2 100644 --- a/src/sources/yfinance.py +++ b/src/sources/yfinance.py @@ -423,8 +423,10 @@ def _is_retired(self, dfq: pd.DataFrame) -> pd.Series: dfq (pd.DataFrame): yfinance questions """ return dfq["id"].map( - lambda qid: self._is_pair_id(qid) - and any(t in self.retired_question_ids for t in self._split_pair_id(qid)) + lambda qid: ( + self._is_pair_id(qid) + and any(t in self.retired_question_ids for t in self._split_pair_id(qid)) + ) ) # ------------------------------------------------------------------ diff --git a/src/tests/leaderboard/test_llm_identities.py b/src/tests/leaderboard/test_llm_identities.py index 23afef06..d57d76cb 100644 --- a/src/tests/leaderboard/test_llm_identities.py +++ b/src/tests/leaderboard/test_llm_identities.py @@ -754,8 +754,7 @@ def test_two_way_fixed_effects_excludes_llm_crowd_comparison_models_from_fit(mon main = import_leaderboard_main() fitted_model_pks = [] crowd_models = [ - "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) " - "geometric mean log odds with news", + "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) geometric mean log odds with news", "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) geometric mean with news", "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) median with news", ] @@ -1041,8 +1040,7 @@ def test_get_df_info_handles_llm_crowd_comparison_models_without_llm_identity(mo messages = [] monkeypatch.setattr(leaderboard_main.slack, "send_message", messages.append, raising=False) comparison_models = [ - "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) " - "geometric mean log odds with news", + "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) geometric mean log odds with news", "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) geometric mean with news", "LLM Crowd (gpt-4o, claude-3.5-sonnet, gemini-1.5-pro) median with news", ] diff --git a/src/tests/llm_forecaster/test_real_question_set_forecast.py b/src/tests/llm_forecaster/test_real_question_set_forecast.py index 78741776..590f801c 100644 --- a/src/tests/llm_forecaster/test_real_question_set_forecast.py +++ b/src/tests/llm_forecaster/test_real_question_set_forecast.py @@ -2,9 +2,8 @@ import json import re -from llm_forecaster import fb_model_runs, output +from llm_forecaster import fb_model_runs, output, runner from llm_forecaster import question_set as question_set_module -from llm_forecaster import runner from llm_forecaster.forecast_variants import ZERO_SHOT, ZERO_SHOT_WITH_FREEZE_VALUES from orchestration import _io, _llm_forecaster_io from sources import DATASET_SOURCE_NAMES diff --git a/src/tests/llm_forecaster/test_runner.py b/src/tests/llm_forecaster/test_runner.py index ce59f99a..b244da37 100644 --- a/src/tests/llm_forecaster/test_runner.py +++ b/src/tests/llm_forecaster/test_runner.py @@ -1158,7 +1158,7 @@ def test_run_model_writes_llm_call_transcript_in_prod_mode(monkeypatch, tmp_path transcript_files = list(tmp_path.glob("*.llm-calls.md")) assert len(transcript_files) == 1 assert transcript_files[0].name == ( - "2026-05-10.ForecastBench.OpenAI.Test Lab." "test-model-run-variant-01.llm-calls.md" + "2026-05-10.ForecastBench.OpenAI.Test Lab.test-model-run-variant-01.llm-calls.md" ) diff --git a/src/tests/orchestration/test_llm_forecaster_io.py b/src/tests/orchestration/test_llm_forecaster_io.py index f8b66df8..fdefe1ba 100644 --- a/src/tests/orchestration/test_llm_forecaster_io.py +++ b/src/tests/orchestration/test_llm_forecaster_io.py @@ -143,7 +143,7 @@ def test_write_final_forecast_file_overwrites_existing_local_file(tmp_path): ), ) existing_path = tmp_path / ( - "TEST.2026-05-10.ForecastBench.OpenAI.Test Lab." "test-model-run-variant-01-zero-shot.json" + "TEST.2026-05-10.ForecastBench.OpenAI.Test Lab.test-model-run-variant-01-zero-shot.json" ) existing_path.write_text("existing", encoding="utf-8") diff --git a/src/tests/orchestration/test_question_set_io.py b/src/tests/orchestration/test_question_set_io.py index 3bf27136..dfe00b3e 100644 --- a/src/tests/orchestration/test_question_set_io.py +++ b/src/tests/orchestration/test_question_set_io.py @@ -100,7 +100,7 @@ def fake_urlopen(url, timeout): assert data["forecast_due_date"] == "2026-05-10" assert calls == [ ( - (f"{_io.DATASETS_QUESTION_SETS_RAW_BASE_URL}" "/2026-05-10-llm.json"), + (f"{_io.DATASETS_QUESTION_SETS_RAW_BASE_URL}/2026-05-10-llm.json"), _io.QUESTION_SET_READ_TIMEOUT_SECONDS, ) ] diff --git a/src/tests/test_fred.py b/src/tests/test_fred.py index 0b92efe0..9e3bbe6b 100644 --- a/src/tests/test_fred.py +++ b/src/tests/test_fred.py @@ -148,9 +148,10 @@ class TestThrottle: def test_sleeps_when_called_too_soon(self, fred_source): """Sleeps the remaining interval when requests are too close together.""" - with patch("sources.fred.time.monotonic", side_effect=[100.1, 100.1]), patch( - "sources.fred.time.sleep" - ) as mock_sleep: + with ( + patch("sources.fred.time.monotonic", side_effect=[100.1, 100.1]), + patch("sources.fred.time.sleep") as mock_sleep, + ): fred_source._last_request_time = 100.0 # elapsed = 0.1 < 0.6 fred_source._throttle() mock_sleep.assert_called_once() @@ -158,9 +159,10 @@ def test_sleeps_when_called_too_soon(self, fred_source): def test_no_sleep_when_interval_elapsed(self, fred_source): """Does not sleep when enough time has already passed.""" - with patch("sources.fred.time.monotonic", side_effect=[101.0, 101.0]), patch( - "sources.fred.time.sleep" - ) as mock_sleep: + with ( + patch("sources.fred.time.monotonic", side_effect=[101.0, 101.0]), + patch("sources.fred.time.sleep") as mock_sleep, + ): fred_source._last_request_time = 100.0 # elapsed = 1.0 > 0.6 fred_source._throttle() mock_sleep.assert_not_called() diff --git a/src/tests/test_kalshi.py b/src/tests/test_kalshi.py index 99f3628a..c9a9852e 100644 --- a/src/tests/test_kalshi.py +++ b/src/tests/test_kalshi.py @@ -149,7 +149,7 @@ def test_resolution_criteria_preserves_stored_sources(self): """A capped fetch cannot remove a previously stored settlement source.""" market = make_kalshi_api_market(rules_primary="Updated primary.", rules_secondary="") existing_criteria = ( - "Old primary. Outcome verified from Library of Congress " "(https://www.congress.gov/)." + "Old primary. Outcome verified from Library of Congress (https://www.congress.gov/)." ) criteria = KalshiSource._resolution_criteria(market, existing_criteria=existing_criteria) @@ -1254,7 +1254,7 @@ def test_existing_unresolved_updated(self, mock_market, mock_build, kalshi_sourc row = result.dfq[result.dfq["id"] == "KXTEST-001"].iloc[0] assert row["question"] == "Updated question text [Yes: Specific Yes outcome]" assert row["market_info_resolution_criteria"] == ( - "New rules Outcome verified from Library of Congress " "(https://www.congress.gov/)." + "New rules Outcome verified from Library of Congress (https://www.congress.gov/)." ) assert row["url"] == "https://kalshi.com/markets/kxtest/x/kxtest" diff --git a/src/tests/test_metaculus.py b/src/tests/test_metaculus.py index 8d7e3bac..614cface 100644 --- a/src/tests/test_metaculus.py +++ b/src/tests/test_metaculus.py @@ -640,7 +640,8 @@ def test_deduplicates_across_categories(self, mock_search, metaculus_source): mock_search.side_effect = [ {"100", "200"}, # base search ] + [ - {"100", "300"} if i == 0 else set() for i in range(14) # 14 categories + {"100", "300"} if i == 0 else set() + for i in range(14) # 14 categories ] dff = metaculus_source.fetch() diff --git a/src/tests/test_runtime_requirements.py b/src/tests/test_runtime_requirements.py index 1bba57e1..713b8327 100644 --- a/src/tests/test_runtime_requirements.py +++ b/src/tests/test_runtime_requirements.py @@ -72,9 +72,9 @@ def test_deploy_requirements_do_not_duplicate_utils_owned_llm_provider_deps(): requirement_names = { re.split(r"[<>=!~]", requirement, maxsplit=1)[0].strip() for requirement in requirements } - assert requirement_names.isdisjoint( - SHARED_LLM_PROVIDER_REQUIREMENT_NAMES - ), requirements_path + assert requirement_names.isdisjoint(SHARED_LLM_PROVIDER_REQUIREMENT_NAMES), ( + requirements_path + ) def test_metadata_deploy_requirements_keep_direct_gcp_deps(): diff --git a/src/tests/test_serpapi.py b/src/tests/test_serpapi.py index e6ae230d..c3d4f57c 100644 --- a/src/tests/test_serpapi.py +++ b/src/tests/test_serpapi.py @@ -355,7 +355,7 @@ def test_update_refreshes_metadata_without_refetching(source, monkeypatch, freez saved = second.dfq.iloc[0] assert saved["resolution_criteria"] == f"Revised criteria for {variables['flight_id']}." assert saved["background"] == ( - f"Revised background for {variables['flight_id']}. " f"See: {saved['url']}" + f"Revised background for {variables['flight_id']}. See: {saved['url']}" ) assert_flight_median_explanation( second.dfq.iloc[0]["freeze_datetime_value_explanation"], diff --git a/src/tests/test_utils_cleanup.py b/src/tests/test_utils_cleanup.py index d6e98617..ce4abd7b 100644 --- a/src/tests/test_utils_cleanup.py +++ b/src/tests/test_utils_cleanup.py @@ -1,9 +1,9 @@ import re import subprocess +import tomllib from pathlib import Path ROOT = Path(__file__).resolve().parents[2] -ROOT_UTILS_EXCLUDE_PATTERN = "| " + "^" + "/utils/" ROOT_UTILS_COPY_COMMAND = " ".join(("cp", "-r", "$(ROOT_DIR)" + "utils")) @@ -23,12 +23,14 @@ def test_root_utils_submodule_metadata_removed(): def test_formatters_no_longer_exclude_root_utils(): - setup_cfg = (ROOT / "setup.cfg").read_text() - pyproject = (ROOT / "pyproject.toml").read_text() + ruff_config = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["ruff"] + exclude_patterns = ( + ruff_config.get("exclude", []) + + ruff_config.get("extend-exclude", []) + + ruff_config.get("format", {}).get("exclude", []) + ) - assert "utils," not in setup_cfg - assert ROOT_UTILS_EXCLUDE_PATTERN not in pyproject - assert "match-dir = ^(?!(\\.venv|utils|" not in setup_cfg + assert not [pattern for pattern in exclude_patterns if "utils" in pattern] def test_deploy_makefiles_do_not_copy_root_utils(): diff --git a/src/tests/test_wikipedia.py b/src/tests/test_wikipedia.py index 97aa3034..0d0fbc85 100644 --- a/src/tests/test_wikipedia.py +++ b/src/tests/test_wikipedia.py @@ -532,11 +532,14 @@ class TestDownloadTables: def test_coerces_value_dtype_and_drops_unparseable_rows(self): page = _fide_elo_page() # fields Player/Rating, value dtype int raw = pd.DataFrame({"Player": ["A", "B"], "Rating": ["2800", "not-a-number"]}) - with patch.object( - WikipediaSource, - "_get_edit_history", - return_value=[(datetime(2024, 6, 1, 12, 0), "rev1")], - ), patch.object(WikipediaSource, "_download_wikipedia_table", return_value=raw.copy()): + with ( + patch.object( + WikipediaSource, + "_get_edit_history", + return_value=[(datetime(2024, 6, 1, 12, 0), "rev1")], + ), + patch.object(WikipediaSource, "_download_wikipedia_table", return_value=raw.copy()), + ): result = WikipediaSource._download_tables(page, session=object()) # "not-a-number" -> NaN -> row dropped; A kept as an int; edit date stamped. @@ -550,9 +553,10 @@ def test_logs_per_page_progress(self, caplog): page = _fide_elo_page() edit_history = [(datetime(2024, 6, 1, 12, 0), f"rev{i}") for i in range(250)] raw = pd.DataFrame({"Player": ["A"], "Rating": ["2800"]}) - with patch.object( - WikipediaSource, "_get_edit_history", return_value=edit_history - ), patch.object(WikipediaSource, "_download_wikipedia_table", return_value=raw.copy()): + with ( + patch.object(WikipediaSource, "_get_edit_history", return_value=edit_history), + patch.object(WikipediaSource, "_download_wikipedia_table", return_value=raw.copy()), + ): with caplog.at_level("INFO", logger="sources.wikipedia"): WikipediaSource._download_tables(page, session=object()) @@ -577,8 +581,9 @@ def test_returns_dict_keyed_by_id_root(self, wikipedia_source): def fake_download_tables(page, session): return fake_df.copy() - with patch.object(WikipediaSource, "_make_session", return_value=object()), patch.object( - WikipediaSource, "_download_tables", side_effect=fake_download_tables + with ( + patch.object(WikipediaSource, "_make_session", return_value=object()), + patch.object(WikipediaSource, "_download_tables", side_effect=fake_download_tables), ): result = wikipedia_source.fetch() @@ -599,8 +604,9 @@ def fake_download_tables(page, session): return pd.DataFrame() return good.copy() - with patch.object(WikipediaSource, "_make_session", return_value=object()), patch.object( - WikipediaSource, "_download_tables", side_effect=fake_download_tables + with ( + patch.object(WikipediaSource, "_make_session", return_value=object()), + patch.object(WikipediaSource, "_download_tables", side_effect=fake_download_tables), ): result = wikipedia_source.fetch() @@ -615,8 +621,9 @@ def fake_download_tables(page, session): ) def test_raises_when_every_page_fails(self, wikipedia_source): - with patch.object(WikipediaSource, "_make_session", return_value=object()), patch.object( - WikipediaSource, "_download_tables", return_value=None + with ( + patch.object(WikipediaSource, "_make_session", return_value=object()), + patch.object(WikipediaSource, "_download_tables", return_value=None), ): with pytest.raises(RuntimeError, match="No Wikipedia data was downloaded"): wikipedia_source.fetch() @@ -629,9 +636,11 @@ def _run_driver(self, failed_pages): from orchestration.func_wikipedia_fetch import main as fetch_main fetch_result = {"FIDE_rankings_elo_rating": pd.DataFrame({"Player": ["A"]})} - with patch.object(fetch_main, "WikipediaSource") as mock_source_class, patch.object( - fetch_main._source_io, "write_wikipedia_fetch_output" - ) as mock_write, patch.object(fetch_main.slack, "send_message") as mock_slack: + with ( + patch.object(fetch_main, "WikipediaSource") as mock_source_class, + patch.object(fetch_main._source_io, "write_wikipedia_fetch_output") as mock_write, + patch.object(fetch_main.slack, "send_message") as mock_slack, + ): mock_source_class.return_value.fetch.return_value = fetch_result mock_source_class.return_value.failed_pages = failed_pages fetch_main.driver(None) @@ -793,14 +802,17 @@ def fake_download(bucket_name, filename, local_filename): f.write(contents[filename]) try: - with patch.object( - _source_io.gcp.storage, - "list_with_prefix", - return_value=[*contents, f"wikipedia/fetch/{prefix}_notes.txt"], - ), patch.object( - _source_io.gcp.storage, - "download_no_error_message_on_404", - side_effect=fake_download, + with ( + patch.object( + _source_io.gcp.storage, + "list_with_prefix", + return_value=[*contents, f"wikipedia/fetch/{prefix}_notes.txt"], + ), + patch.object( + _source_io.gcp.storage, + "download_no_error_message_on_404", + side_effect=fake_download, + ), ): result = _source_io.read_wikipedia_fetch_files() finally: