--- quick_sorter.py +++ quick_sorter.py @@ -93,6 +93,7 @@ detail: str = "" version: str = "" collision: str = "" # exact duplicate / different-content variant handling + duplicate_of: str = "" # full-SHA-256 match found by the global pre-pass @property def target(self) -> Path: @@ -738,6 +739,93 @@ return False +def global_duplicate_prepass(records: list[Record], source_root: Path, target_root: Path) -> dict[str, int | bool | str]: + """Mark exact duplicates before content extraction or LLM analysis. + + The pre-pass is deliberately enabled only for *separate* source and target + trees. This covers migration/staging imports without changing the established + V2 Inbox behaviour where the Inbox is nested inside a broader target root. + + Full SHA-256 is used for every exact-duplicate decision. Existing target files + are authoritative. If no target copy exists, the first source copy becomes the + representative and later byte-identical source copies are marked duplicates. + """ + stats: dict[str, int | bool | str] = { + "enabled": False, + "target_files": 0, + "source_files": 0, + "target_duplicates": 0, + "source_duplicates": 0, + "hash_errors": 0, + "reason": "", + } + + source_root = source_root.resolve() + target_root = target_root.resolve() + + if source_root == target_root or source_root.is_relative_to(target_root): + stats["reason"] = "source is the target or nested inside it; preserving normal V2 behaviour" + return stats + if target_root.is_relative_to(source_root): + stats["reason"] = "target is nested inside source; unsafe layout for global dedupe" + return stats + + stats["enabled"] = True + target_by_hash: dict[str, Path] = {} + + for p in sorted(target_root.rglob("*")): + if p.is_symlink() or not p.is_file() or ignored(p, target_root): + continue + try: + digest = sha256_file(p) + except OSError: + stats["hash_errors"] += 1 + continue + stats["target_files"] += 1 + target_by_hash.setdefault(digest, p) + + source_seen: dict[str, Path] = {} + for rec in records: + # scan() has already applied the normal ignore rules. Archives are hashed + # too: an archive that is already byte-identical in TARGET needs no LLM. + try: + digest = sha256_file(rec.path) + except OSError: + stats["hash_errors"] += 1 + continue + stats["source_files"] += 1 + + target_match = target_by_hash.get(digest) + if target_match is not None: + rec.duplicate_of = str(target_match) + rec.status = "exact-duplicate" + rec.collision = "EXACT → already in target" + try: + display = str(target_match.relative_to(target_root)) + except ValueError: + display = str(target_match) + rec.reason = f"SHA-256 match in target: {display}" + stats["target_duplicates"] += 1 + continue + + first_source = source_seen.get(digest) + if first_source is not None: + rec.duplicate_of = str(first_source) + rec.status = "exact-duplicate" + rec.collision = "EXACT → duplicate incoming" + try: + display = str(first_source.relative_to(source_root)) + except ValueError: + display = str(first_source) + rec.reason = f"SHA-256 match in source: {display}" + stats["source_duplicates"] += 1 + continue + + source_seen[digest] = rec.path + + return stats + + def variant_path(target: Path, n: int) -> Path: """Insert a visible variant marker before the full extension chain.""" suffix = "".join(target.suffixes) @@ -785,6 +873,29 @@ if rec.status != "confirmed": continue source = rec.path + + # A global pre-pass duplicate bypasses semantic destination/name planning. + # Re-verify the bytes immediately before planning deletion; if the other + # copy changed or vanished, fail closed and touch nothing. + if rec.duplicate_of: + duplicate_of = Path(rec.duplicate_of) + if duplicate_of.exists() and same_content(source, duplicate_of): + result = "EXACT duplicate -> would remove source" + resolved = duplicate_of + collision = "EXACT → remove duplicate" + else: + result = "SKIPPED - pre-pass duplicate no longer matches" + resolved = duplicate_of + collision = "RECHECK → duplicate changed" + rec.collision = collision + rows.append({ + "old_path": str(source), + "new_path": str(resolved), + "requested_path": str(rec.target), + "result": result, + }) + continue + target = rec.target if target == source: @@ -980,10 +1091,11 @@ return [(iid, self.by_iid[iid]) for iid in self.tree.selection()] def recompute_collisions(self): - # Full hashes are calculated only for rows that actually compete for - # the same target, so this stays cheap in normal use. + # Preserve global pre-pass markers on rows that have not yet been + # confirmed. plan() still handles ordinary same-target collisions. for rec in self.records: - rec.collision = "" + if not rec.duplicate_of: + rec.collision = "" plan(self.records) for iid in self.by_iid: self.refresh(iid) @@ -1007,6 +1119,7 @@ merged.append(rec) new_count += 1 self.records = merged + global_duplicate_prepass(self.records, root, target_root) self._populate() messagebox.showinfo( "Rescan complete", @@ -1018,10 +1131,20 @@ messagebox.showinfo("Busy", "Analysis is already running.") return selected = [r for _, r in self.selected()] - chosen = [r for r in selected if r.status != "archive" and not is_archive_file(r.path)] - archive_count = len(selected) - len(chosen) + duplicate_count = sum(1 for r in selected if r.duplicate_of) + archive_count = sum(1 for r in selected if not r.duplicate_of and (r.status == "archive" or is_archive_file(r.path))) + chosen = [ + r for r in selected + if not r.duplicate_of and r.status != "archive" and not is_archive_file(r.path) + ] if not chosen: - if archive_count: + if duplicate_count: + messagebox.showinfo( + "Exact duplicate selected", + f"{duplicate_count} exact SHA-256 duplicate(s) are already resolved and are not sent to the LLM.\n" + "Confirm them only when you want Apply to remove the redundant SOURCE copy.", + ) + elif archive_count: messagebox.showinfo( "Archive selected", "Archive files are intentionally not sent to the LLM.\n" @@ -1261,12 +1384,22 @@ root, target_root, destinations, aliases=aliases, allow_stay_outside_policy=inbox_mode, ) + dedupe = global_duplicate_prepass(records, root, target_root) print(f"Root: {root}") print(f"Target root: {target_root}") print(f"Policy profile: {profile}") print(f"Files: {len(records)}") print(f"Archives: {sum(r.status == 'archive' for r in records)}") + if dedupe["enabled"]: + print( + "Global SHA-256 pre-pass: " + f"target files={dedupe['target_files']}, source files={dedupe['source_files']}, " + f"already in target={dedupe['target_duplicates']}, " + f"duplicate incoming={dedupe['source_duplicates']}, hash errors={dedupe['hash_errors']}" + ) + else: + print(f"Global SHA-256 pre-pass: disabled ({dedupe['reason']})") print(f"Allowed destinations: {len(destinations)}") print(f"Strong route aliases: {sum(len(v) for v in aliases.values())} across {len(aliases)} branches") print(f"Date from filename: {sum(r.date_source == 'filename' for r in records)}") @@ -1275,7 +1408,8 @@ if args.scan_only: for r in records[:25]: - print(f"{r.date_guess} [{r.date_source:8}] {r.rel_path}") + duplicate = f" -> {r.duplicate_of}" if r.duplicate_of else "" + print(f"{r.status:15} {r.date_guess} [{r.date_source:8}] {r.rel_path}{duplicate}") return launch_gui(root, target_root, records, destinations, args.endpoint, args.model, max(1, args.batch_size), aliases=aliases, inbox_mode=inbox_mode)