214 lines
8.9 KiB
Diff
214 lines
8.9 KiB
Diff
--- quick_sorter.py
|
|
+++ quick_sorter.py
|
|
@@ -93,6 +93,7 @@
|
|
detail: str = ""
|
|
version: str = ""
|
|
collision: str = "" # exact duplicate / different-content variant handling
|
|
+ duplicate_of: str = "" # full-SHA-256 match found by the global pre-pass
|
|
|
|
@property
|
|
def target(self) -> Path:
|
|
@@ -738,6 +739,93 @@
|
|
return False
|
|
|
|
|
|
+def global_duplicate_prepass(records: list[Record], source_root: Path, target_root: Path) -> dict[str, int | bool | str]:
|
|
+ """Mark exact duplicates before content extraction or LLM analysis.
|
|
+
|
|
+ The pre-pass is deliberately enabled only for *separate* source and target
|
|
+ trees. This covers migration/staging imports without changing the established
|
|
+ V2 Inbox behaviour where the Inbox is nested inside a broader target root.
|
|
+
|
|
+ Full SHA-256 is used for every exact-duplicate decision. Existing target files
|
|
+ are authoritative. If no target copy exists, the first source copy becomes the
|
|
+ representative and later byte-identical source copies are marked duplicates.
|
|
+ """
|
|
+ stats: dict[str, int | bool | str] = {
|
|
+ "enabled": False,
|
|
+ "target_files": 0,
|
|
+ "source_files": 0,
|
|
+ "target_duplicates": 0,
|
|
+ "source_duplicates": 0,
|
|
+ "hash_errors": 0,
|
|
+ "reason": "",
|
|
+ }
|
|
+
|
|
+ source_root = source_root.resolve()
|
|
+ target_root = target_root.resolve()
|
|
+
|
|
+ if source_root == target_root or source_root.is_relative_to(target_root):
|
|
+ stats["reason"] = "source is the target or nested inside it; preserving normal V2 behaviour"
|
|
+ return stats
|
|
+ if target_root.is_relative_to(source_root):
|
|
+ stats["reason"] = "target is nested inside source; unsafe layout for global dedupe"
|
|
+ return stats
|
|
+
|
|
+ stats["enabled"] = True
|
|
+ target_by_hash: dict[str, Path] = {}
|
|
+
|
|
+ for p in sorted(target_root.rglob("*")):
|
|
+ if p.is_symlink() or not p.is_file() or ignored(p, target_root):
|
|
+ continue
|
|
+ try:
|
|
+ digest = sha256_file(p)
|
|
+ except OSError:
|
|
+ stats["hash_errors"] += 1
|
|
+ continue
|
|
+ stats["target_files"] += 1
|
|
+ target_by_hash.setdefault(digest, p)
|
|
+
|
|
+ source_seen: dict[str, Path] = {}
|
|
+ for rec in records:
|
|
+ # scan() has already applied the normal ignore rules. Archives are hashed
|
|
+ # too: an archive that is already byte-identical in TARGET needs no LLM.
|
|
+ try:
|
|
+ digest = sha256_file(rec.path)
|
|
+ except OSError:
|
|
+ stats["hash_errors"] += 1
|
|
+ continue
|
|
+ stats["source_files"] += 1
|
|
+
|
|
+ target_match = target_by_hash.get(digest)
|
|
+ if target_match is not None:
|
|
+ rec.duplicate_of = str(target_match)
|
|
+ rec.status = "exact-duplicate"
|
|
+ rec.collision = "EXACT → already in target"
|
|
+ try:
|
|
+ display = str(target_match.relative_to(target_root))
|
|
+ except ValueError:
|
|
+ display = str(target_match)
|
|
+ rec.reason = f"SHA-256 match in target: {display}"
|
|
+ stats["target_duplicates"] += 1
|
|
+ continue
|
|
+
|
|
+ first_source = source_seen.get(digest)
|
|
+ if first_source is not None:
|
|
+ rec.duplicate_of = str(first_source)
|
|
+ rec.status = "exact-duplicate"
|
|
+ rec.collision = "EXACT → duplicate incoming"
|
|
+ try:
|
|
+ display = str(first_source.relative_to(source_root))
|
|
+ except ValueError:
|
|
+ display = str(first_source)
|
|
+ rec.reason = f"SHA-256 match in source: {display}"
|
|
+ stats["source_duplicates"] += 1
|
|
+ continue
|
|
+
|
|
+ source_seen[digest] = rec.path
|
|
+
|
|
+ return stats
|
|
+
|
|
+
|
|
def variant_path(target: Path, n: int) -> Path:
|
|
"""Insert a visible variant marker before the full extension chain."""
|
|
suffix = "".join(target.suffixes)
|
|
@@ -785,6 +873,29 @@
|
|
if rec.status != "confirmed":
|
|
continue
|
|
source = rec.path
|
|
+
|
|
+ # A global pre-pass duplicate bypasses semantic destination/name planning.
|
|
+ # Re-verify the bytes immediately before planning deletion; if the other
|
|
+ # copy changed or vanished, fail closed and touch nothing.
|
|
+ if rec.duplicate_of:
|
|
+ duplicate_of = Path(rec.duplicate_of)
|
|
+ if duplicate_of.exists() and same_content(source, duplicate_of):
|
|
+ result = "EXACT duplicate -> would remove source"
|
|
+ resolved = duplicate_of
|
|
+ collision = "EXACT → remove duplicate"
|
|
+ else:
|
|
+ result = "SKIPPED - pre-pass duplicate no longer matches"
|
|
+ resolved = duplicate_of
|
|
+ collision = "RECHECK → duplicate changed"
|
|
+ rec.collision = collision
|
|
+ rows.append({
|
|
+ "old_path": str(source),
|
|
+ "new_path": str(resolved),
|
|
+ "requested_path": str(rec.target),
|
|
+ "result": result,
|
|
+ })
|
|
+ continue
|
|
+
|
|
target = rec.target
|
|
|
|
if target == source:
|
|
@@ -980,10 +1091,11 @@
|
|
return [(iid, self.by_iid[iid]) for iid in self.tree.selection()]
|
|
|
|
def recompute_collisions(self):
|
|
- # Full hashes are calculated only for rows that actually compete for
|
|
- # the same target, so this stays cheap in normal use.
|
|
+ # Preserve global pre-pass markers on rows that have not yet been
|
|
+ # confirmed. plan() still handles ordinary same-target collisions.
|
|
for rec in self.records:
|
|
- rec.collision = ""
|
|
+ if not rec.duplicate_of:
|
|
+ rec.collision = ""
|
|
plan(self.records)
|
|
for iid in self.by_iid:
|
|
self.refresh(iid)
|
|
@@ -1007,6 +1119,7 @@
|
|
merged.append(rec)
|
|
new_count += 1
|
|
self.records = merged
|
|
+ global_duplicate_prepass(self.records, root, target_root)
|
|
self._populate()
|
|
messagebox.showinfo(
|
|
"Rescan complete",
|
|
@@ -1018,10 +1131,20 @@
|
|
messagebox.showinfo("Busy", "Analysis is already running.")
|
|
return
|
|
selected = [r for _, r in self.selected()]
|
|
- chosen = [r for r in selected if r.status != "archive" and not is_archive_file(r.path)]
|
|
- archive_count = len(selected) - len(chosen)
|
|
+ duplicate_count = sum(1 for r in selected if r.duplicate_of)
|
|
+ archive_count = sum(1 for r in selected if not r.duplicate_of and (r.status == "archive" or is_archive_file(r.path)))
|
|
+ chosen = [
|
|
+ r for r in selected
|
|
+ if not r.duplicate_of and r.status != "archive" and not is_archive_file(r.path)
|
|
+ ]
|
|
if not chosen:
|
|
- if archive_count:
|
|
+ if duplicate_count:
|
|
+ messagebox.showinfo(
|
|
+ "Exact duplicate selected",
|
|
+ f"{duplicate_count} exact SHA-256 duplicate(s) are already resolved and are not sent to the LLM.\n"
|
|
+ "Confirm them only when you want Apply to remove the redundant SOURCE copy.",
|
|
+ )
|
|
+ elif archive_count:
|
|
messagebox.showinfo(
|
|
"Archive selected",
|
|
"Archive files are intentionally not sent to the LLM.\n"
|
|
@@ -1261,12 +1384,22 @@
|
|
root, target_root, destinations, aliases=aliases,
|
|
allow_stay_outside_policy=inbox_mode,
|
|
)
|
|
+ dedupe = global_duplicate_prepass(records, root, target_root)
|
|
|
|
print(f"Root: {root}")
|
|
print(f"Target root: {target_root}")
|
|
print(f"Policy profile: {profile}")
|
|
print(f"Files: {len(records)}")
|
|
print(f"Archives: {sum(r.status == 'archive' for r in records)}")
|
|
+ if dedupe["enabled"]:
|
|
+ print(
|
|
+ "Global SHA-256 pre-pass: "
|
|
+ f"target files={dedupe['target_files']}, source files={dedupe['source_files']}, "
|
|
+ f"already in target={dedupe['target_duplicates']}, "
|
|
+ f"duplicate incoming={dedupe['source_duplicates']}, hash errors={dedupe['hash_errors']}"
|
|
+ )
|
|
+ else:
|
|
+ print(f"Global SHA-256 pre-pass: disabled ({dedupe['reason']})")
|
|
print(f"Allowed destinations: {len(destinations)}")
|
|
print(f"Strong route aliases: {sum(len(v) for v in aliases.values())} across {len(aliases)} branches")
|
|
print(f"Date from filename: {sum(r.date_source == 'filename' for r in records)}")
|
|
@@ -1275,7 +1408,8 @@
|
|
|
|
if args.scan_only:
|
|
for r in records[:25]:
|
|
- print(f"{r.date_guess} [{r.date_source:8}] {r.rel_path}")
|
|
+ duplicate = f" -> {r.duplicate_of}" if r.duplicate_of else ""
|
|
+ print(f"{r.status:15} {r.date_guess} [{r.date_source:8}] {r.rel_path}{duplicate}")
|
|
return
|
|
|
|
launch_gui(root, target_root, records, destinations, args.endpoint, args.model, max(1, args.batch_size), aliases=aliases, inbox_mode=inbox_mode)
|