build: replace black, isort, flake8 and pydocstyle with ruff

.gitignore

@@ -58,8 +58,8 @@ ENV/
 # notebook checkpoints
 .ipynb_checkpoints/
 
-# isort
-*.py.isorted
+# ruff
+.ruff_cache/
 
 # OS generated files
 .DS_Store

AGENTS.md

@@ -102,12 +102,10 @@ NB: `variables.mk` contains both runtime environment variables and variables tha
 
 ## Code style
 
-* Formatting: black (line length 100)
-* Import sorting: isort (black-compatible profile)
-* Linting: flake8
-* Docstrings: pydocstyle
-* Run `make lint` to apply formatting and check all of the above
-* Configuration is in `pyproject.toml` and `setup.cfg`
+* Formatting and linting: ruff (`ruff format` for formatting, `ruff check` for import sorting, lint
+  rules and docstrings), line length 100
+* Run `make lint` to apply formatting and auto-fixes, and check all of the above
+* Configuration is in `pyproject.toml`
 * Functions should use type hints
 * New ForecastBench Python files should not add `from __future__ import annotations`
 * Prefer f-strings for Python string interpolation, including log messages. Use logger

Makefile

@@ -46,19 +46,18 @@ export DEFAULT_CLOUD_FUNCTION_ENV_VARS=CLOUD_PROJECT=$(CLOUD_PROJECT),GOOGLE_CLO
 .PHONY: all clean lint test deploy install-requirements setup-python-env
 
 MAKE_LINT_ERROR_OUT ?= 0
-ISORT_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check-only,)
-BLACK_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check,)
+RUFF_CHECK_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),,--fix --exit-zero)
+RUFF_FORMAT_FLAGS := $(if $(filter 1,$(MAKE_LINT_ERROR_OUT)),--check,)
 VENV_PYTHON := .venv/bin/python
 ROOT_REQUIREMENTS_STAMP := .venv/.root-requirements-installed
 ALL_REQUIREMENTS_STAMP := .venv/.all-requirements-installed
 ROOT_REQUIREMENTS_NO_RUNTIME := .venv/.root-requirements-no-runtime.txt
 SRC_REQUIREMENTS := $(shell find src -type f -name requirements.txt ! -path '*/upload/*')
 
 lint:
-	isort $(ISORT_FLAGS) .
-	black $(BLACK_FLAGS) .
-	flake8 .
-	pydocstyle .
+	ruff check $(RUFF_CHECK_FLAGS) .
+	ruff format $(RUFF_FORMAT_FLAGS) .
+	ruff check .
 
 test: setup-python-env
 	@. ${ROOT_DIR}.venv/bin/activate && python -m pytest src/tests/ $(ARGS)

paper/category_table/main.py

@@ -7,7 +7,7 @@
 from variables import BUCKET_NAME
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
-from src.helpers import resolution  # noqa: E402
+from src.helpers import resolution
 
 dfm = pd.read_json(
     f"gs://{BUCKET_NAME}/question_metadata.jsonl",

paper/llm_breakdown_tables/main.py

@@ -15,7 +15,7 @@
 )
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
-from src.helpers import resolution  # noqa: E402
+from src.helpers import resolution
 
 
 def make_list_hashable(df, col):

paper/source_table/main.py

@@ -8,7 +8,7 @@
 from variables import BUCKET_NAME
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
-from src.helpers import resolution  # noqa: E402
+from src.helpers import resolution
 
 question_counts = {}
 df = pd.DataFrame()
@@ -110,4 +110,4 @@ def get_source_values(df, source_name):
     \multicolumn{{2}}{{l}}{{\textbf{{Question Bank Total}}}} & {total_n} & {total_pairs} \\
     \bottomrule
   \end{{tabular}}
-""")  # noqa: B950
+""")  # noqa: E501

pyproject.toml

@@ -2,12 +2,34 @@
 testpaths = ["src/tests"]
 pythonpath = ["src"]
 
-[tool.black]
+[tool.ruff]
 line-length = 100
-force-exclude = '''
-(
-    ^/.venv.*
-    | ^/experiments/stability-analysis/
-    | ^/experiments/ranking-simulation/
-)
-'''
+target-version = "py314"
+src = [".", "src"]
+extend-exclude = [
+    ".venv*",
+    "experiments/stability-analysis",
+    "experiments/ranking-simulation",
+    "upload*",
+]
+
+[tool.ruff.lint]
+# Parity with the previous isort + flake8 (+ bugbear B950) + pydocstyle setup.
+select = ["E", "W", "F", "B", "I", "D"]
+# E203/E701 are formatter-owned; B9xx is bugbear's opt-in set (B950 is emulated by E501 below).
+ignore = ["E203", "E701", "B9"]
+
+[tool.ruff.lint.pycodestyle]
+# E501 at 110 reproduces bugbear's B950 (line-length + 10 %).
+max-line-length = 110
+
+[tool.ruff.lint.pydocstyle]
+convention = "pep257"
+
+[tool.ruff.lint.per-file-ignores]
+"**/tests/**" = ["D"]
+"src/sources/fred_questions.py" = ["E501"]
+"src/sources/wikipedia.py" = ["E501"]
+
+[tool.ruff.format]
+exclude = ["*.md"]

requirements.txt

@@ -1,8 +1,4 @@
 -r requirements.runtime.txt
-isort==6.0.0
-black[jupyter]==26.3.1
-flake8==7.1.2
-flake8-bugbear==24.12.12
-pydocstyle==6.3.0
+ruff==0.17.1
 pytest==9.0.3
 pytest-mock==3.15.1

scripts/generate_yfinance_pairs.py

@@ -171,8 +171,7 @@ def main() -> None:
         "--seed",
         type=int,
         default=DEFAULT_SEED,
-        help="Seed for drawing new pairs; existing pairs are never redrawn "
-        "(default: %(default)s).",
+        help="Seed for drawing new pairs; existing pairs are never redrawn (default: %(default)s).",
     )
     args = parser.parse_args()
     if not args.question_file.is_file():

src/base_eval/experiments/llm_crowd_candidate_testing.ipynb

@@ -157,7 +157,6 @@
     "def executor(max_workers, prompt, model_name, save_dict):\n",
     "    \"\"\"Scratchpad executor.\"\"\"\n",
     "    with ThreadPoolExecutor(max_workers=max_workers) as executor:\n",
-    "\n",
     "        worker_with_args = partial(\n",
     "            worker, prompt=prompt, model_name=model_name, save_dict=save_dict\n",
     "        )\n",
@@ -207,9 +206,9 @@
     "            results[model] = {}\n",
     "        try:\n",
     "            results[model] = read_jsonl(file_path)\n",
-    "            model_result_loaded[model][\n",
-    "                f\"prompt_{prompt_index}\"\n",
-    "            ] = True  # Set flag to True if loaded successfully\n",
+    "            model_result_loaded[model][f\"prompt_{prompt_index}\"] = (\n",
+    "                True  # Set flag to True if loaded successfully\n",
+    "            )\n",
     "        except:\n",
     "            results[model][f\"prompt_{prompt_index}\"] = {i: \"\" for i in range(len(questions_list))}\n",
     "\n",

src/base_eval/experiments/scratchpad_testing.ipynb

@@ -411,7 +411,6 @@
     "def executor(max_workers, prompt, model_name, save_dict):\n",
     "    \"\"\"Scratchpad executor.\"\"\"\n",
     "    with ThreadPoolExecutor(max_workers=max_workers) as executor:\n",
-    "\n",
     "        worker_with_args = partial(\n",
     "            worker, prompt=prompt, model_name=model_name, save_dict=save_dict\n",
     "        )\n",
@@ -445,9 +444,9 @@
     "            results[model] = {}\n",
     "        try:\n",
     "            results[model] = read_jsonl(file_path)\n",
-    "            model_result_loaded[model][\n",
-    "                f\"prompt_{prompt_index}\"\n",
-    "            ] = True  # Set flag to True if loaded successfully\n",
+    "            model_result_loaded[model][f\"prompt_{prompt_index}\"] = (\n",
+    "                True  # Set flag to True if loaded successfully\n",
+    "            )\n",
     "        except:\n",
     "            results[model][f\"prompt_{prompt_index}\"] = {i: \"\" for i in range(len(questions_list))}\n",
     "\n",

src/base_eval/naive_and_dummy_forecasters/main.py

@@ -14,7 +14,7 @@
 from utils import gcp
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))
-from helpers import (  # noqa: E402
+from helpers import (
     acled,
     constants,
     data_utils,
@@ -24,7 +24,7 @@
     wikipedia,
     yfinance,
 )
-from orchestration import _io  # noqa: E402
+from orchestration import _io
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -392,7 +392,7 @@ def prepare_df_and_set_null_values(df, forecast_due_date, last_date_for_data):
 
     # Expand resolution dates
     df["resolution_dates"] = df.apply(
-        lambda x: ([] if x["source"] in resolution.MARKET_SOURCES else x["resolution_dates"]),
+        lambda x: [] if x["source"] in resolution.MARKET_SOURCES else x["resolution_dates"],
         axis=1,
     )
     df = df.explode("resolution_dates", ignore_index=True)

src/helpers/dbnomics.py

@@ -5,7 +5,7 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
-from sources._metadata import SOURCE_METADATA  # noqa: E402
+from sources._metadata import SOURCE_METADATA
 
 _META = SOURCE_METADATA["dbnomics"]
 SOURCE_INTRO = _META["source_intro"]

src/helpers/fred.py

@@ -10,7 +10,7 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
-from sources._metadata import SOURCE_METADATA  # noqa: E402
+from sources._metadata import SOURCE_METADATA
 
 _META = SOURCE_METADATA["fred"]
 SOURCE_INTRO = _META["source_intro"]

src/helpers/metadata_prompts.py

@@ -66,4 +66,4 @@
 Insert thinking:
 {{ insert your concise thoughts here }}
 Classification:
-{{ insert "flag" or "ok"}}"""  # noqa: B950
+{{ insert "flag" or "ok"}}"""  # noqa: E501

src/helpers/polymarket.py

@@ -5,7 +5,7 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
 
-from sources._metadata import SOURCE_METADATA  # noqa: E402
+from sources._metadata import SOURCE_METADATA
 
 _META = SOURCE_METADATA["polymarket"]
 SOURCE_INTRO = _META["source_intro"]

src/helpers/question_curation.py

@@ -28,9 +28,9 @@
 FREEZE_QUESTION_SOURCES = {**FREEZE_QUESTION_MARKET_SOURCES, **FREEZE_QUESTION_DATA_SOURCES}
 
 for _sources in (FREEZE_QUESTION_MARKET_SOURCES, FREEZE_QUESTION_DATA_SOURCES):
-    assert math.isclose(
-        sum(source["sampling_share"] for source in _sources.values()), 1
-    ), "sampling shares within a source type must sum to 1"
+    assert math.isclose(sum(source["sampling_share"] for source in _sources.values()), 1), (
+        "sampling shares within a source type must sum to 1"
+    )
 
 DATA_SOURCES = list(FREEZE_QUESTION_DATA_SOURCES.keys())
 MARKET_SOURCES = list(FREEZE_QUESTION_MARKET_SOURCES.keys())

src/helpers/slack.py

@@ -5,7 +5,7 @@
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
 
-from . import keys  # noqa: E402
+from . import keys
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)

src/helpers/wikipedia.py

@@ -23,11 +23,11 @@
 # Private imports from the source module are deliberate: this helper only exists until
 # `base_eval` is refactored to call `WikipediaSource.get_naive_forecast()`, after which it is
 # deleted and nothing outside `sources.wikipedia` needs these.
-from sources.wikipedia import _PAGES as PAGES  # noqa: E402
-from sources.wikipedia import (  # noqa: E402
+from sources.wikipedia import _PAGES as PAGES
+from sources.wikipedia import (
     _WIKIPEDIA_QUESTION_BANK_DATA_STORAGE_START_DATE as WIKIPEDIA_QUESTION_BANK_DATA_STORAGE_START_DATE,
 )
-from sources.wikipedia import QuestionType  # noqa: E402
+from sources.wikipedia import QuestionType
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)