Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,8 @@ mapillary_tools check_upload_history MY_CAPTURE_DIR \
It prints a JSON array with one entry per image sequence, video or zip file, which tells whether it was already
uploaded (see the
[command reference](https://github.com/mapillary/mapillary_tools/blob/main/docs/command_reference.md#check_upload_history)).
It reads only the local upload history: it needs no login, makes no network requests, and changes nothing.
It reads only the local upload history: it needs no login and makes no network requests.
It writes only the checksums of the files, to the cache that `upload` uses, so that a later check or upload does not read them again.

# Advanced Usage

Expand Down
9 changes: 5 additions & 4 deletions docs/command_reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -371,8 +371,8 @@ Gotchas:
video is recognized by its file alone, even if its track changed. The upload history is kept on this computer
only. `--dry_run` neither reads nor writes it.
- To look files up in the history, `upload` reads each one in full to compute its checksum, with a progress bar.
The checksums are cached in the upload history folder by file path, size and modification time, so the next
run skips uploaded files without reading them.
The checksums are cached in the upload history folder by file path, size, modification time, change time and
file ID, so the next run skips uploaded files without reading them.
- An interrupted upload resumes where it stopped the next time you upload the same files.
- `--organization_key` takes a numeric ID. Organization names are 2 to 32 letters, digits, `-` and `_`, starting
with a letter, and are case-sensitive. Saving a name again with another ID replaces it.
Expand Down Expand Up @@ -502,8 +502,9 @@ video or zip file: its file type, sequence UUID, checksum, file names, `already_
`already_uploaded`, which is true only when all its files are in the history. Images are looked up one by one,
also in sequences uploaded before, so it can report as uploaded a sequence that `upload` would upload again.

It reads only the local upload history: it needs no login, makes no network requests, and changes nothing.
Uploads from another computer, or with `--dry_run`, are not in it.
It reads only the local upload history: it needs no login and makes no network requests. Uploads from another
computer, or with `--dry_run`, are not in it. It writes only the checksums of the files, to the cache that `upload`
uses, so that a later check or upload does not read them again.

## zip

Expand Down
44 changes: 34 additions & 10 deletions mapillary_tools/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,7 @@ def upload(
)
# Cache the checksums that the history is looked up by, so that files
# already uploaded are not read again to be skipped
checksum_cache = uploader.create_checksum_cache(
Path(constants.MAPILLARY_UPLOAD_HISTORY_PATH).joinpath("checksum_cache")
)
checksum_cache = _create_checksum_cache()

# Set up tdqm
_setup_tdqm(emitter)
Expand Down Expand Up @@ -186,28 +184,49 @@ def _has_upload_history_record(md5sum: str) -> bool:
return isinstance(record, dict)


def _create_checksum_cache() -> history.PersistentCache | None:
return uploader.create_checksum_cache(
Path(constants.MAPILLARY_UPLOAD_HISTORY_PATH).joinpath("checksum_cache")
)


def check_upload_history(
import_path: Path | T.Sequence[Path],
desc_path: str | None = None,
skip_subfolders: bool = False,
_metadatas_from_process: T.Sequence[types.MetadataOrError] | None = None,
) -> list[UploadHistoryCheckResult]:
"""Return upload-history matches without uploading or changing history."""
"""
Return upload-history matches without uploading or changing history.

The checksums of the files are taken from, and computed ones written to,
the checksum cache that upload uses, so that neither a later check nor
the upload reads the files again to compute them. Nothing else is written.
"""
import_paths = _normalize_import_paths(import_path)
metadatas = _load_descs(_metadatas_from_process, import_paths, desc_path)
candidates = _find_upload_candidates(
metadatas, import_paths, skip_subfolders=skip_subfolders
)

# As upload, use the cache only with the upload history
checksum_cache: history.PersistentCache | None = None
if not _is_history_disabled(dry_run=False):
checksum_cache = _create_checksum_cache()

results: list[UploadHistoryCheckResult] = []
image_results_to_check: list[
tuple[UploadHistoryCheckResult, list[types.ImageMetadata]]
] = []
image_md5sums_to_check: set[str] = set()

for sequence_uuid, sequence in types.group_and_sort_images(
candidates.image_metadatas
).items():
sequences = types.group_and_sort_images(candidates.image_metadatas)
for sequence_idx, (sequence_uuid, sequence) in enumerate(sequences.items()):
uploader._update_md5sums(
sequence,
desc=f"Checksumming IMAGE ({sequence_idx + 1}/{len(sequences)})",
checksum_cache=checksum_cache,
)
sequence_md5sum = types.update_sequence_md5sum(sequence)
filenames = [metadata.filename for metadata in sequence]
sequence_already_uploaded = _has_upload_history_record(sequence_md5sum)
Expand Down Expand Up @@ -243,10 +262,15 @@ def check_upload_history(
result["filenames"]
)

for video_metadata in sorted(
sorted_video_metadatas = sorted(
candidates.video_metadatas, key=lambda metadata: metadata.filename
):
video_metadata.update_md5sum()
)
for idx, video_metadata in enumerate(sorted_video_metadatas):
uploader._update_md5sums(
[video_metadata],
desc=f"Checksumming {video_metadata.filetype.value.upper()} {video_metadata.filename.name} ({idx + 1}/{len(sorted_video_metadatas)})",
checksum_cache=checksum_cache,
)
assert isinstance(video_metadata.md5sum, str), "md5sum should be calculated"
filenames = [video_metadata.filename]
already_uploaded_filenames = (
Expand Down
51 changes: 45 additions & 6 deletions mapillary_tools/uploader.py
Original file line number Diff line number Diff line change
Expand Up @@ -1155,10 +1155,35 @@ def create_checksum_cache(path: Path) -> history.PersistentCache | None:


def _checksum_cache_key(path: Path) -> str:
# A file whose path, size and modification time are unchanged is taken
# as unchanged, as rsync does by default
# A file whose path, size, modification time, change time and file ID are
# unchanged is taken as unchanged, as git does. Size and modification time
# alone are not enough: editing the GPS of an image with exiftool -P keeps
# both, and the checksum cached for the old contents would then match the
# upload history of the old image, so the edited image would be skipped
# as already uploaded. The system sets the change time on every change,
# and it can not be set back; a file replaced by another gets a new file
# ID. On Windows, st_ctime is the creation time: there only the file ID
# tells a replaced file, on NTFS. FAT and exFAT, as on memory cards, store
# no change time, and Windows makes their file IDs from the position of
# the file in its folder, which the replacing file can take, so an edit
# there that keeps the size and modification time may not be told
stat = path.stat()
return json.dumps([os.path.abspath(path), stat.st_size, stat.st_mtime_ns])
return json.dumps(
[
os.path.abspath(path),
stat.st_size,
stat.st_mtime_ns,
stat.st_ctime_ns,
stat.st_ino,
]
)


def _checksum_cache_key_or_none(path: Path) -> str | None:
try:
return _checksum_cache_key(path)
except OSError:
return None


def _update_md5sums(
Expand All @@ -1173,7 +1198,9 @@ def _update_md5sums(

With a checksum cache, take the checksums of the files from it, and cache
the checksums computed, so that uploading the files again, as when they
were already uploaded, does not read them again.
were already uploaded, does not read them again. A file that changed
while it was read is not cached, as its checksum may be of the contents
before or during the change.
"""
missing = [metadata for metadata in metadatas if metadata.md5sum is None]
if not missing:
Expand All @@ -1200,16 +1227,28 @@ def _update_md5sums(
unit_divisor=1024,
disable=LOG.isEnabledFor(logging.DEBUG),
) as pbar:
unchanged: list[types.ImageMetadata | types.VideoMetadata] = []
for metadata in missing:
with metadata.filename.open("rb") as fp:
metadata.md5sum = utils.md5sum_fp(fp, progress=pbar.update).hexdigest()
if checksum_cache is not None:
if (
_checksum_cache_key_or_none(metadata.filename)
== cache_keys[metadata.filename]
):
unchanged.append(metadata)
else:
LOG.debug(
"Not caching the checksum of %s, changed while read",
metadata.filename,
)

if checksum_cache is not None:
if checksum_cache is not None and unchanged:
try:
checksum_cache.set_many(
{
cache_keys[metadata.filename]: T.cast(str, metadata.md5sum)
for metadata in missing
for metadata in unchanged
},
expires_in=CHECKSUM_CACHE_EXPIRES_IN,
)
Expand Down
9 changes: 8 additions & 1 deletion tests/integration/test_history.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,14 @@ def test_check_upload_history_is_read_only_and_matches_upload(
results = run_check_upload_history(command_args)
assert results
assert all(not result["already_uploaded"] for result in results)
assert not Path(os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"]).exists()
# No upload history is written: only the checksum cache, which upload
# uses too
history_path = Path(os.environ["MAPILLARY_UPLOAD_HISTORY_PATH"])
assert [path.name for path in history_path.iterdir()]
assert all(
path.is_file() and path.name.startswith("checksum_cache")
for path in history_path.iterdir()
)

run_process_and_upload_for_descs(process_args)

Expand Down
Loading
Loading