774- token = f" {needle} " 775- if token in filename_text or token in folder_text or token in source_text or token in excerpt_text: 776- matched_branches.add(branch) 777- break 778- 779- if len(matched_branches) == 1: 780- return next(iter(matched_branches)) 781- return "" 782- 783- 784-def choose_candidates(rec: Record, destinations: list[str], limit: int = 14, 785- aliases: Optional[dict[str, list[str]]] = None, 786: allow_stay_outside_policy: bool = False) -> list[str]: 787- if not destinations: 788- return [rec.current_folder] if rec.current_folder else [""] 789- 790- # User-defined strong aliases route obvious domain shorthand before the LLM. 791- branch = _strong_route_branch(rec, aliases or {}) 792- if branch: 793- routed = [d for d in destinations if d == branch or d.startswith(branch + "/")] 794- if routed: 795- destinations = routed 796- 797- text = f"{rec.filename} {rec.current_folder} {rec.source_context} {rec.excerpt[:400]}" 798- q = tokens(text) -- 821- for top in top_names: 822- if top in destinations and top not in chosen: 823- chosen.append(top) 824- 825- for _, folder in scored: 826- if folder not in chosen: 827- chosen.append(folder) 828- if len(chosen) >= limit: 829- break 830- 831- # For a real 01_INBOX we must always allow "leave it where it is" when the 832- # model cannot know enough. This can be a nested unpack folder as well. 833: if allow_stay_outside_policy and rec.current_folder and rec.current_folder not in chosen: 834- chosen.append(rec.current_folder) 835- 836: return chosen[:limit + (1 if allow_stay_outside_policy else 0)] 837- 838- 839-def scan(root: Path, target_root: Path, destinations: list[str], 840- aliases: Optional[dict[str, list[str]]] = None, 841: allow_stay_outside_policy: bool = False) -> list[Record]: 842- records = [] 843- for p in sorted(root.rglob("*")): 844- if not p.is_file() or ignored(p, root): 845- continue 846- rel = str(p.relative_to(root)) 847- try: 848: current = str(p.parent.relative_to(target_root)) 849- except ValueError: 850- current = str(p.parent.relative_to(root)) 851- if current == ".": 852- current = "" 853- # Keep provenance separate from destination logic. This matters when an 854- # incoming backup root itself carries useful ownership context (for 855- # example a Cafe-Bach backup) that would otherwise disappear when paths 856- # are made relative to the source root. 857- source_context = root.name 858- relative_parent = str(p.parent.relative_to(root)) 859- if relative_parent != ".": 860- source_context = f"{root.name}/{relative_parent}" -- 870- current_folder=current, 871- date_guess=d, 872- date_source=source, 873- proposed_name=p.name, 874- destination=current, 875- status="archive" if archive else "pending", 876- content_status="archive" if archive else "not-read", 877- reason=("Archiv: entpacken, Inhalt sortieren, bei Bedarf sauber neu packen" if archive else ""), 878- source_context=source_context, 879- ) 880- rec.candidates = choose_candidates( 881- rec, destinations, aliases=aliases, 882: allow_stay_outside_policy=allow_stay_outside_policy, 883- ) 884- records.append(rec) 885- return records 886- 887- 888-# ---------------- LLM batching ---------------- 889- 890-SYSTEM_PROMPT = """You are a conservative file-sorting assistant. 891-Your job is to save the human time, not create more decisions. 892- 893-Return one JSON ARRAY only, one object per input item, in the same order. 894-Each object must contain exactly: -- 997- if _result_confidence(result) < 0.85: 998- return True 999- if is_generic_filename(rec.filename): 1000- return True 1001- return False 1002- 1003- 1004-def llm_batches( 1005- records: list[Record], 1006- endpoint: str, 1007- model: str, 1008- batch_size: int = 1, 1009: fallback_model: str = "", 1010-): 1011- """Analyse exactly one file per LLM request. 1012- 1013- batch_size is retained for CLI/backward compatibility but intentionally 1014- ignored: one-file requests prevent cross-file influence and isolate failures. 1015- 1016- When a fallback model is configured, it is used only for a primary error, 1017- an invalid destination, confidence below 0.85, or a generic filename. 1018- If primary and fallback disagree on destination, the primary proposal is 1019- kept but confidence is capped at 0.69 so the GUI forces human review. 1020- """ 1021- del batch_size 1022: fallback_model = fallback_model.strip() 1023- 1024- for rec in records: 1025- batch = [rec] 1026- primary = None 1027- primary_error = None 1028- try: 1029- primary = _call_llm_one(rec, endpoint, model) 1030- except Exception as exc: 1031- primary_error = str(exc) 1032- 1033- use_fallback = ( 1034: bool(fallback_model) 1035: and fallback_model != model 1036- and (primary_error is not None or _needs_fallback(rec, primary)) 1037- ) 1038- 1039- if not use_fallback: 1040- if primary_error is not None: 1041- yield batch, None, primary_error 1042- else: 1043- yield batch, [primary], None 1044- continue 1045- 1046- try: 1047: fallback = _call_llm_one(rec, endpoint, fallback_model) 1048- except Exception as exc: 1049- fallback_error = str(exc) 1050- if primary is not None: 1051- # A usable primary answer is better than throwing it away merely 1052- # because the optional second opinion failed. 1053- yield batch, [primary], None 1054- else: 1055- yield batch, None, f"primary={primary_error}; fallback={fallback_error}" 1056- continue 1057- 1058- if primary is None: 1059- chosen = dict(fallback) 1060: chosen["reason"] = f"{fallback_model} fallback: {chosen.get('reason', '')}".strip() 1061- yield batch, [chosen], None 1062- continue 1063- 1064- primary_dest = str(primary.get("destination", "")).strip() 1065- fallback_dest = str(fallback.get("destination", "")).strip() 1066- 1067- if primary_dest == fallback_dest: 1068- # For the cases deliberately escalated, prefer the second opinion 1069- # when both models agree on the actual destination. 1070- chosen = dict(fallback) 1071: chosen["reason"] = f"{fallback_model} fallback: {chosen.get('reason', '')}".strip() 1072- yield batch, [chosen], None 1073- else: 1074- # Do not silently choose between disagreeing models. Keep the fast 1075- # primary proposal visible, but force explicit human review. 1076- chosen = dict(primary) 1077- chosen["confidence"] = min(_result_confidence(primary), 0.69) 1078- chosen["reason"] = ( 1079- f"Model disagreement: {model}={primary_dest}; " 1080: f"{fallback_model}={fallback_dest}" 1081- ) 1082- yield batch, [chosen], None 1083- 1084- 1085-def extract_json_array(text: str): 1086- text = text.strip() 1087- if text.startswith("```"): 1088- text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.I) 1089- text = re.sub(r"\s*```$", "", text) 1090- try: 1091- return json.loads(text) 1092- except json.JSONDecodeError: -- 1191- "enabled": False, 1192- "target_files": 0, 1193- "source_files": 0, 1194- "target_duplicates": 0, 1195- "source_duplicates": 0, 1196- "hash_errors": 0, 1197- "reason": "", 1198- } 1199- 1200- source_root = source_root.resolve() 1201- target_root = target_root.resolve() 1202- 1203: if source_root == target_root or source_root.is_relative_to(target_root): 1204- stats["reason"] = "source is the target or nested inside it; preserving normal V2 behaviour" 1205- return stats 1206- if target_root.is_relative_to(source_root): 1207- stats["reason"] = "target is nested inside source; unsafe layout for global dedupe" 1208- return stats 1209- 1210- stats["enabled"] = True 1211- target_by_hash: dict[str, Path] = {} 1212- 1213- for p in sorted(target_root.rglob("*")): 1214- if p.is_symlink() or not p.is_file() or ignored(p, target_root): 1215- continue -- 1227- # too: an archive that is already byte-identical in TARGET needs no LLM. 1228- try: 1229- digest = sha256_file(rec.path) 1230- except OSError: 1231- stats["hash_errors"] += 1 1232- continue 1233- stats["source_files"] += 1 1234- 1235- target_match = target_by_hash.get(digest) 1236- if target_match is not None: 1237- rec.duplicate_of = str(target_match) 1238- rec.status = "exact-duplicate" 1239: rec.collision = "EXACT → already in target" 1240- try: 1241: display = str(target_match.relative_to(target_root)) 1242- except ValueError: 1243- display = str(target_match) 1244- rec.reason = f"SHA-256 match in target: {display}" 1245- stats["target_duplicates"] += 1 1246- continue 1247- 1248- first_source = source_seen.get(digest) 1249- if first_source is not None: 1250- rec.duplicate_of = str(first_source) 1251- rec.status = "exact-duplicate" 1252- rec.collision = "EXACT → duplicate incoming" 1253- try: -- 1447- if out: 1448- write_header = not log_path.exists() 1449- with log_path.open("a", newline="", encoding="utf-8") as f: 1450- writer = csv.DictWriter(f, fieldnames=out[0].keys()) 1451- if write_header: 1452- writer.writeheader() 1453- writer.writerows(out) 1454- return out 1455- 1456- 1457-# ---------------- GUI ---------------- 1458- 1459:def launch_gui(root: Path, target_root: Path, records: list[Record], destinations: list[str], endpoint: str, model: str, batch_size: int, fallback_model: str = "", aliases: Optional[dict[str, list[str]]] = None, inbox_mode: bool = False): 1460- import tkinter as tk 1461- from tkinter import ttk, messagebox 1462- 1463- class App: 1464- def __init__(self, win): 1465- self.win = win 1466- self.records = records 1467- self.destinations = destinations 1468- self.by_iid = {} 1469- self.q = queue.Queue() 1470- self.analysis_running = False 1471- self.active_editor = None -- 1535- rec.collision = "" 1536- plan(self.records) 1537- for iid in self.by_iid: 1538- self.refresh(iid) 1539- 1540- def rescan(self): 1541- if self.analysis_running: 1542- messagebox.showinfo("Busy", "Analysis is already running.") 1543- return 1544- old = {str(r.path): r for r in self.records} 1545- fresh = scan( 1546- root, target_root, self.destinations, aliases=aliases, 1547: allow_stay_outside_policy=inbox_mode, 1548- ) 1549- merged = [] 1550- new_count = 0 1551- for rec in fresh: 1552- previous = old.get(str(rec.path)) 1553- if previous is not None: 1554- merged.append(previous) 1555- else: 1556- merged.append(rec) 1557- new_count += 1 1558- self.records = merged 1559- global_duplicate_prepass(self.records, root, target_root) -- 1589- ) 1590- else: 1591- messagebox.showinfo("Select files", "Select one or more rows first.") 1592- return 1593- self.analysis_running = True 1594- self.status.config(text=f"Analysing {len(chosen)} file(s), one request per file…") 1595- 1596- def worker(): 1597- for rec in chosen: 1598- if not rec.excerpt: 1599- rec.excerpt, rec.content_status = extract_excerpt(rec.path) 1600- rec.candidates = choose_candidates(rec, self.destinations, aliases=aliases) 1601: for batch, results, error in llm_batches(chosen, endpoint, model, batch_size, fallback_model=fallback_model): 1602- self.q.put((batch, results, error)) 1603- self.q.put((None, None, "__DONE__")) 1604- 1605- threading.Thread(target=worker, daemon=True).start() 1606- self.win.after(100, self.poll) 1607- 1608- def poll(self): 1609- try: 1610- while True: 1611- batch, results, error = self.q.get_nowait() 1612- if error == "__DONE__": 1613- self.analysis_running = False -- 1729- "No existing file will be overwritten. Exact duplicates are only " 1730- "removed after a full-file SHA-256 match." 1731- ) 1732- if not messagebox.askyesno("Apply resolved batch?", summary): 1733- return 1734- result = apply_confirmed(self.records, root / LOG_FILENAME) 1735- moved = sum(1 for x in result if x["result"] in ("moved", "moved as variant")) 1736- removed = sum(1 for x in result if x["result"] == "removed exact duplicate") 1737- skipped = sum(1 for x in result if x["result"].startswith("SKIPPED")) 1738- if inbox_mode: 1739- self.records = scan( 1740- root, target_root, self.destinations, aliases=aliases, 1741: allow_stay_outside_policy=True, 1742- ) 1743- self._populate() 1744- messagebox.showinfo( 1745- "Done", 1746- f"Moved/renamed: {moved}\nRemoved exact duplicates: {removed}\nSkipped: {skipped}\n" 1747- f"Remaining in Inbox: {len(self.records) if inbox_mode else 'n/a'}\nLog: {root / LOG_FILENAME}" 1748- ) 1749- 1750- def sort(self, column, reverse): 1751- rows = [(self.tree.set(iid, column), iid) for iid in self.tree.get_children("")] 1752- rows.sort(key=lambda x: x[0].lower(), reverse=reverse) 1753- for pos, (_, iid) in enumerate(rows): -- 1813- target_root = Path(args.target_root).expanduser().resolve() 1814- elif inbox_mode: 1815- target_root = root.parent 1816- else: 1817- target_root = root 1818- if not target_root.is_dir(): 1819- raise SystemExit(f"Not a target directory: {target_root}") 1820- 1821- destinations = allowed_destinations(root, policy, args.profile) 1822- aliases = policy.get("_strong_aliases", {}).get(profile, {}) 1823- records = scan( 1824- root, target_root, destinations, aliases=aliases, 1825: allow_stay_outside_policy=inbox_mode, 1826- ) 1827- dedupe = global_duplicate_prepass(records, root, target_root) 1828- 1829- print(f"Root: {root}") 1830- print(f"Target root: {target_root}") 1831- print(f"Policy profile: {profile}") 1832- print(f"Files: {len(records)}") 1833- print(f"Archives: {sum(r.status == 'archive' for r in records)}") 1834- if dedupe["enabled"]: 1835- print( 1836: "Global SHA-256 pre-pass: " 1837- f"target files={dedupe['target_files']}, source files={dedupe['source_files']}, " 1838: f"already in target={dedupe['target_duplicates']}, " 1839- f"duplicate incoming={dedupe['source_duplicates']}, hash errors={dedupe['hash_errors']}" 1840- ) 1841- else: 1842: print(f"Global SHA-256 pre-pass: disabled ({dedupe['reason']})") 1843- print(f"Allowed destinations: {len(destinations)}") 1844- print(f"Strong route aliases: {sum(len(v) for v in aliases.values())} across {len(aliases)} branches") 1845- print(f"Date from filename: {sum(r.date_source == 'filename' for r in records)}") 1846- print(f"Date from dated parent folder: {sum(r.date_source == 'folder' for r in records)}") 1847- print(f"Date from mtime fallback: {sum(r.date_source == 'mtime' for r in records)}") 1848- 1849: fallback_model = "" if args.no_fallback else args.fallback_model.strip() 1850- print(f"Primary LLM model: {args.model}") 1851: print(f"Fallback LLM model: {fallback_model or 'disabled'}") 1852- 1853- if args.scan_only: 1854- for r in records[:25]: 1855- duplicate = f" -> {r.duplicate_of}" if r.duplicate_of else "" 1856- print(f"{r.status:15} {r.date_guess} [{r.date_source:8}] {r.rel_path}{duplicate}") 1857- return 1858- 1859: launch_gui(root, target_root, records, destinations, args.endpoint, args.model, max(1, args.batch_size), fallback_model=fallback_model, aliases=aliases, inbox_mode=inbox_mode) 1860- 1861- 1862-if __name__ == "__main__": 1863- main()