Skip to content

fix: enforce single concurrent migration with atomic cross-row lock #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
thehecktour wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: enforce single concurrent migration with atomic cross-row lock #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ade73a
fix: pass uuid4() call result instead of function object in rollback …
thehecktour Jun 21, 2026
6058147
fix: enforce single concurrent migration with atomic cross-row lock
thehecktour Jun 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 30 additions & 9 deletions housewatch/async_migrations/async_migration_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,38 @@ def execute_op(sql: str, args=None, *, query_id: str, timeout_seconds: int = 600


def mark_async_migration_as_running(migration: AsyncMigration) -> bool:
# update to running iff the state was Starting (ui triggered) or NotStarted (api triggered)
with transaction.atomic():
if migration.status not in [MigrationStatus.Starting, MigrationStatus.NotStarted]:
# Cross-row check: lock all rows in Running or Starting state before proceeding.
# This blocks any concurrent transaction attempting the same check, ensuring only
# one migration transitions to Running at a time across the entire table.
already_running = (
AsyncMigration.objects
.select_for_update()
.filter(status__in=[MigrationStatus.Running, MigrationStatus.Starting])
.exclude(pk=migration.pk)
.exists()
)
if already_running:
logger.warning(
"Refusing to start migration: another migration is already running or starting",
migration=migration.name,
)
return False

# Re-read this migration's own row under a lock to guard against double-start
# of the same migration by two concurrent workers.
instance = AsyncMigration.objects.select_for_update().get(pk=migration.pk)
if instance.status not in [MigrationStatus.Starting, MigrationStatus.NotStarted]:
return False
migration.status = MigrationStatus.Running
migration.current_query_id = ""
migration.progress = 0
migration.current_operation_index = 0
migration.started_at = now()
migration.finished_at = None
migration.save()

instance.status = MigrationStatus.Running
instance.current_query_id = ""
instance.progress = 0
instance.current_operation_index = 0
instance.started_at = now()
instance.finished_at = None
instance.save()

return True


Expand Down
7 changes: 5 additions & 2 deletions housewatch/async_migrations/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,10 @@ def start_async_migration(migration: AsyncMigration, ignore_posthog_version=Fals

if not mark_async_migration_as_running(migration):
# we don't want to touch the migration, i.e. don't process_error
logger.error(f"Migration state has unexpectedly changed for async migration {migration.name}")
logger.error(
"Could not start migration: another migration is already running or migration state changed unexpectedly",
migration=migration.name,
)
return False

return run_async_migration_operations(migration)
Expand Down Expand Up @@ -157,7 +160,7 @@ def attempt_migration_rollback(migration: AsyncMigration):
op = ops[op_index]
if not op:
continue
execute_op(op, query_id=str(uuid4))
execute_op(op, query_id=str(uuid4()))
except Exception as e:
last_error = f"At operation {op_index} rollback failed with error:{str(e)}"
process_error(
Expand Down