diff --git a/src/sidecar/main.py b/src/sidecar/main.py index db7f48b..f9fc356 100644 --- a/src/sidecar/main.py +++ b/src/sidecar/main.py @@ -61,11 +61,6 @@ def run_sync_cycle(): current_cycle_files = set() - # Prune uploaded_files set to only include files that currently exist - # This prevents the set from growing indefinitely. - existing_files = {str(p) for p in root.glob("**/*.m4s")} - uploaded_files.intersection_update(existing_files) - # Use ThreadPoolExecutor to parallelize uploads with ThreadPoolExecutor(max_workers=10) as executor: for app_dir in root.iterdir(): @@ -83,6 +78,7 @@ def run_sync_cycle(): elif entry.name.endswith('.m4s'): p = Path(entry.path) segments.append(p) + # Track file for pruning uploaded_files at the end of cycle current_cycle_files.add(str(p)) # Sort segments by name to enable binary search @@ -118,6 +114,7 @@ def run_sync_cycle(): executor.submit(upload_file, segment, s3_key_seg) # Prune uploaded_files set to only include files that currently exist + # (Using files tracked during the main loop iteration) uploaded_files.intersection_update(current_cycle_files) except Exception as e: