43 lines
1.6 KiB
Python
43 lines
1.6 KiB
Python
"""
|
|
A file's path changes the moment you rename or move it — which means path
|
|
can't be used as the key for "have I already handled this file?" across
|
|
sessions. A file processed today and reopened next week needs to be
|
|
recognised at its NEW location, not just its old one.
|
|
|
|
Identity here is (size, partial content hash) — cheap even for large
|
|
files, since only the first chunk is read, not the whole thing.
|
|
"""
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
|
|
_CHUNK_SIZE = 65536
|
|
|
|
|
|
def file_identity(path: Path) -> str:
|
|
size = path.stat().st_size
|
|
h = hashlib.sha1()
|
|
try:
|
|
with open(path, "rb") as f:
|
|
h.update(f.read(_CHUNK_SIZE))
|
|
except OSError:
|
|
pass # fall back to size-only identity if the file can't be read
|
|
return f"{size}:{h.hexdigest()}"
|
|
|
|
|
|
def find_duplicate_identities(records) -> dict:
|
|
"""Groups records that share an identity — genuinely byte-identical
|
|
files (same size, same content), not files that merely look similar.
|
|
|
|
This is a real, unavoidable limitation: content-based identity cannot
|
|
tell two identical files apart, because there is nothing to tell apart
|
|
— their content really is the same. Progress-tracking (which file was
|
|
already handled) can silently apply to the wrong twin when this
|
|
happens. Callers should surface this to the person rather than hide
|
|
it; the real fix is deduplicating genuinely identical files as a
|
|
separate pass, not a cleverer identity function.
|
|
"""
|
|
groups: dict = {}
|
|
for rec in records:
|
|
groups.setdefault(rec.identity, []).append(rec)
|
|
return {identity: recs for identity, recs in groups.items() if len(recs) > 1}
|