Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/reference/api/defining_and_executing_simulations.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ Executing Simulations
TaskStatusDB
get_task_df
get_dependency_df
update_max_tries
storage.FileSystemWarehouse


Expand Down
1 change: 1 addition & 0 deletions docs/reference/cli/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ Execution Commands

run_task
status
update_task_db
quickrun

Results Gathering Commands
Expand Down
7 changes: 7 additions & 0 deletions docs/reference/cli/update_task_db.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
.. _cli_update_task_db:

``openfe update-task-db``
=========================

.. click:: openfecli.commands.update_task_db:update_task_db
:prog: openfe update-task-db
23 changes: 23 additions & 0 deletions news/max_tries.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
**Added:**

* Added ``openfe.update_max_tries()`` for updating a ``TaskStatusDB`` (`PR #2227 <https://github.com/OpenFreeEnergy/openfe/pull/2227/>`_).

**Changed:**

* <news item>

**Deprecated:**

* <news item>

**Removed:**

* <news item>

**Fixed:**

* <news item>

**Security:**

* <news item>
2 changes: 1 addition & 1 deletion src/openfe/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
)

from . import analysis, orchestration, setup, storage, utils
from .orchestration import get_dependency_df, get_task_df, setup_task_campaign
from .orchestration import get_dependency_df, get_task_df, setup_task_campaign, update_max_tries
from .orchestration.worker import Worker
from .setup import (
KartografAtomMapper,
Expand Down
1 change: 1 addition & 0 deletions src/openfe/orchestration/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,6 @@
get_dependency_df,
get_task_df,
setup_task_campaign,
update_max_tries,
)
from .worker import Worker
53 changes: 53 additions & 0 deletions src/openfe/orchestration/exorcist_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@
structures and can initialize an Exorcist task database from that graph.
"""

from datetime import datetime
from pathlib import Path

import exorcist
import networkx as nx
import pandas as pd
from annotated_types import Annotated, Gt
from gufe import AlchemicalNetwork, ProtocolDAG

from ..storage.warehouse import FileSystemWarehouse, WarehouseBaseClass
Expand Down Expand Up @@ -153,3 +155,54 @@ def get_dependency_df(task_db: exorcist.TaskStatusDB) -> pd.DataFrame:
"""

return pd.read_sql_table("dependencies", task_db.engine)


def update_max_tries(task_db: exorcist.TaskStatusDB, max_tries: Annotated[int, Gt(0)]):
"""Update the "max_tries" column to `max_tries`.
Only rows that do _not_ have status=COMPLETED will be operated on.

Parameters
----------
task_db : openfe.TaskStatusDB
The TaskStatusDB to update
max_tries : int
The positive integer value to assign as ``max_tries`` for the updated columns.

Raises
------
ValueError: If max_tries is not a positive integer.

"""

import sqlalchemy as sqla
from exorcist.models import TaskStatus

# TODO: select a single task_id?
Comment thread
atravitz marked this conversation as resolved.
Resolved
if type(max_tries) is not int or max_tries <= 0:
raise ValueError("`max_tries` must be a positive integer.")

values = {"max_tries": max_tries, "last_modified": datetime.now()}

# on the rows that already hit TOO_MANY_RETRIES, only allow increasing max_tries,
# otherwise the state is invalid, and set to AVAILABLE
stmt_update_too_many_retries = (
sqla.update(task_db.tasks_table)
.where(task_db.tasks_table.c.status == TaskStatus.TOO_MANY_RETRIES.value)
.where(task_db.tasks_table.c.max_tries < max_tries)
Comment thread
atravitz marked this conversation as resolved.
Resolved
.values(**values, status=TaskStatus.AVAILABLE.value)
)

# COMPLETED tasks should not be updated, and never allow max_tries to be less than
# the number of attempts already made (tries), otherwise the state is invalid.
stmt_update_uncompleted_tasks = (
Comment thread
atravitz marked this conversation as resolved.
Resolved
sqla.update(task_db.tasks_table)
.where(task_db.tasks_table.c.status != TaskStatus.COMPLETED.value)
.where(task_db.tasks_table.c.tries < max_tries)
Comment thread
atravitz marked this conversation as resolved.
Resolved
.where(task_db.tasks_table.c.max_tries != max_tries)
.values(**values)
)

with task_db.engine.begin() as conn:
# TODO: is there a better way than calling this twice?
conn.execute(stmt_update_too_many_retries)
conn.execute(stmt_update_uncompleted_tasks)
86 changes: 85 additions & 1 deletion src/openfe/tests/orchestration/test_exorcist_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
get_task_df,
setup_task_campaign,
)
from openfe.orchestration.exorcist_utils import _alchemical_network_to_task_graph
from openfe.orchestration.exorcist_utils import _alchemical_network_to_task_graph, update_max_tries
from openfe.storage.warehouse import WarehouseBaseClass


Expand Down Expand Up @@ -268,3 +268,87 @@ def test_get_dependency_df(benzene_star_map_task_db):
assert isinstance(df, pd.DataFrame)
assert list(df.columns) == ["from", "to", "blocking"]
assert len(df) == 504


# NOTE: pulled this fixture from exorcist, since I expect to upstream the tests below
from datetime import datetime
Comment thread
atravitz marked this conversation as resolved.
Resolved

from exorcist import TaskStatus, TaskStatusDB

_DEFAULT_DATETIME = datetime(1970, 1, 1)


def patch_exorcist_datetime(with_datetime=_DEFAULT_DATETIME):
# turns out we can't patch just the now() method (datetime is immutable,
# probably C code?) so we have to patch the entire datetime module
loc = "exorcist.taskdb.datetime"
datetime_now = mock.Mock(now=mock.Mock(return_value=with_datetime))
return mock.patch(loc, datetime_now)


def patch_openfe_datetime(with_datetime=_DEFAULT_DATETIME):
loc = "openfe.orchestration.exorcist_utils.datetime"
datetime_now = mock.Mock(now=mock.Mock(return_value=with_datetime))
return mock.patch(loc, datetime_now)


def get_tasks_and_deps(db):
with db.engine.connect() as conn:
tasks = set(conn.execute(sqla.select(db.tasks_table)))
deps = set(conn.execute(sqla.select(db.dependencies_table)))
return tasks, deps


def task_row(taskid, status, last_modified, tries, max_tries, task_type=""):
return (taskid, status.value, last_modified, tries, max_tries, task_type)


@pytest.fixture
def fresh_db():
# create an empty database
echo = False # switch this for debugging
engine = sqla.create_engine("sqlite://", echo=echo)

yield TaskStatusDB(engine)


def test_update_max_tries(fresh_db):
task_db = fresh_db
task_db.add_task("task_a", requirements=[], max_tries=1)
task_db.add_task("task_b", requirements=["task_a"], max_tries=1)
task_db.add_task("task_c", requirements=[], max_tries=1)

with patch_exorcist_datetime():
taskid = task_db.check_out_task()
assert taskid == "task_a"
task_db.mark_task_completed(taskid, success=True)

taskid = task_db.check_out_task()
assert taskid == "task_b"
task_db.mark_task_completed(taskid, success=False)

tasks, deps = get_tasks_and_deps(task_db)

assert tasks == {
task_row("task_a", TaskStatus.COMPLETED, _DEFAULT_DATETIME, 1, 1, ""),
task_row("task_b", TaskStatus.TOO_MANY_RETRIES, _DEFAULT_DATETIME, 1, 1, ""),
task_row("task_c", TaskStatus.AVAILABLE, None, 0, 1, ""),
}
assert deps == {("task_a", "task_b", False)}

datetime2 = datetime(1999, 1, 1)
with patch_openfe_datetime(datetime2):
update_max_tries(task_db=task_db, max_tries=18)
tasks, deps = get_tasks_and_deps(task_db)

assert tasks == {
task_row("task_a", TaskStatus.COMPLETED, _DEFAULT_DATETIME, 1, 1, ""),
task_row("task_b", TaskStatus.AVAILABLE, datetime2, 1, 18, ""),
task_row("task_c", TaskStatus.AVAILABLE, datetime2, 0, 18, ""),
}


@pytest.mark.parametrize("bad_value", ["1", -2, 3.0, 0, True])
def test_update_max_tries_invalid_val(fresh_db, bad_value):
with pytest.raises(ValueError, match="must be a positive integer"):
update_max_tries(task_db=fresh_db, max_tries=bad_value)
106 changes: 106 additions & 0 deletions src/openfecli/commands/update_task_db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
# This code is part of OpenFE and is licensed under the MIT license.
# For details, see https://github.com/OpenFreeEnergy/openfe

from pathlib import Path

import click

from openfecli import OFECommandPlugin
from openfecli.utils import write


def update_task_db_main(
task_db_path: Path,
max_tries: int,
):
"""
Parameters
----------
task_db_path : pathlib.Path
Path to a task.db

max_tries: int
Value to update "max_tries" column to, for applicable rows


Example
-------
>>> openfe status --task-db task.db

┌─────────────────────────────┬──────────────────┬─────────────────────┬───────┬───────────┐
│ task_id │ status │ last_modified │ tries │ max_tries │
├─────────────────────────────┼──────────────────┼─────────────────────┼───────┼───────────┤
│ SetupUnit-d568ebe569b445c7… │ COMPLETED │ 2026-08-14 11:19:16 │ 1 │ 3 │
│ SetupUnit-17dc05e0d79747e7… │ COMPLETED │ 2026-08-14 11:19:21 │ 1 │ 3 │
│ SetupUnit-78321eda905c4c2c… │ COMPLETED │ 2026-08-14 11:19:22 │ 1 │ 3 │
│ SetupUnit-0238f65b55044b1e… │ COMPLETED │ 2026-08-14 11:19:22 │ 1 │ 3 │
│ MultiStateSimulationUnit-2… │ COMPLETED │ 2026-08-14 11:25:10 │ 1 │ 3 │
│ MultiStateSimulationUnit-1… │ COMPLETED │ 2026-08-14 11:25:34 │ 1 │ 3 │
│ MultiStateSimulationUnit-f… │ COMPLETED │ 2026-08-14 11:26:22 │ 1 │ 3 │
│ MultiStateSimulationUnit-c… │ TOO_MANY_RETRIES │ 2026-08-14 11:26:24 │ 3 │ 3 │
│ MultiStateAnalysisUnit-a72… │ COMPLETED │ 2026-08-14 11:26:51 │ 1 │ 3 │
│ MultiStateAnalysisUnit-e44… │ COMPLETED │ 2026-08-14 11:26:26 │ 1 │ 3 │
│ MultiStateAnalysisUnit-7e9… │ COMPLETED │ 2026-08-14 11:29:07 │ 1 │ 3 │
│ MultiStateAnalysisUnit-72c… │ BLOCKED │ NaT │ 0 │ 3 │
└─────────────────────────────┴──────────────────┴─────────────────────┴───────┴───────────┘

>>> openfe update-task-db task.db --max-tries=6
┌─────────────────────────────┬──────────────────┬─────────────────────┬───────┬───────────┐
│ task_id │ status │ last_modified │ tries │ max_tries │
├─────────────────────────────┼──────────────────┼─────────────────────┼───────┼───────────┤
│ SetupUnit-d568ebe569b445c7… │ COMPLETED │ 2026-08-14 11:19:16 │ 1 │ 3 │
│ SetupUnit-17dc05e0d79747e7… │ COMPLETED │ 2026-08-14 11:19:21 │ 1 │ 3 │
│ SetupUnit-78321eda905c4c2c… │ COMPLETED │ 2026-08-14 11:19:22 │ 1 │ 3 │
│ SetupUnit-0238f65b55044b1e… │ COMPLETED │ 2026-08-14 11:19:22 │ 1 │ 3 │
│ MultiStateSimulationUnit-2… │ COMPLETED │ 2026-08-14 11:25:10 │ 1 │ 3 │
│ MultiStateSimulationUnit-1… │ COMPLETED │ 2026-08-14 11:25:34 │ 1 │ 3 │
│ MultiStateSimulationUnit-f… │ COMPLETED │ 2026-08-14 11:26:22 │ 1 │ 3 │
│ MultiStateSimulationUnit-c… │ AVAILABLE │ 2026-08-14 11:27:24 │ 3 │ 6 │
│ MultiStateAnalysisUnit-a72… │ COMPLETED │ 2026-08-14 11:26:51 │ 1 │ 3 │
│ MultiStateAnalysisUnit-e44… │ COMPLETED │ 2026-08-14 11:26:26 │ 1 │ 3 │
│ MultiStateAnalysisUnit-7e9… │ COMPLETED │ 2026-08-14 11:29:07 │ 1 │ 3 │
│ MultiStateAnalysisUnit-72c… │ BLOCKED │ NaT │ 0 │ 6 │
└─────────────────────────────┴──────────────────┴─────────────────────┴───────┴───────────┘

"""

from exorcist import TaskStatusDB

from openfe.orchestration.exorcist_utils import update_max_tries

task_db = TaskStatusDB.from_filename(task_db_path)
update_max_tries(task_db, max_tries)


@click.command("update-task-db", short_help="Update the 'max_tries' column.")
@click.option(
"--task-db",
type=click.Path(
exists=True,
readable=True,
file_okay=True,
dir_okay=False,
path_type=Path,
),
required=True,
help="Path to a TaskDB file.",
)
@click.option(
"--max-tries",
type=click.IntRange(min=1),
required=True,
help="The positive integer value to assign as 'max_tries' for the updated columns.",
)
# NOTE: this is named intentionally broad so that we can add "update_task_type" in a future version
def update_task_db(task_db: Path, max_tries: int):
"""
Update the 'max_tries' column for applicable tasks. COMPLETED tasks and tasks with tries > the input 'max-tries' will not be updated.

"""

# TODO: add loading bar
write("Loading task db ...")
update_task_db_main(task_db_path=task_db, max_tries=max_tries)


PLUGIN = OFECommandPlugin(command=update_task_db, section="Execution", requires_ofe=(1, 13))
40 changes: 40 additions & 0 deletions src/openfecli/tests/commands/test_update_task_db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import exorcist
import networkx as nx
import pytest
import sqlalchemy as sqla
from click.testing import CliRunner

from openfecli.commands.update_task_db import update_task_db

from ..utils import assert_click_success


@pytest.fixture
def simple_task_graph():
task_graph = nx.DiGraph()
node_ids = ["HybridTopologySetupUnit-123", "HybridTopologySetupUnit-456"]

for id in node_ids:
task_graph.add_node(id)

return task_graph, node_ids


def test_update_task_db(simple_task_graph):
runner = CliRunner()
with runner.isolated_filesystem():
task_graph, _ = simple_task_graph
db_path = "test.db"
db = exorcist.TaskStatusDB.from_filename(db_path)
db.add_task_network(task_graph, max_tries=2)
input_max_tries = 7
result = runner.invoke(
update_task_db, ["--task-db", db_path, "--max-tries", str(input_max_tries)]
)
assert_click_success(result)

# just make sure the table is actually modified. all edge cases etc. are handled in the Python API tests
with db.engine.connect() as conn:
new_max_tries = set(conn.execute(sqla.select(db.tasks_table.c.max_tries)))

assert new_max_tries == {(input_max_tries,)}
Loading