399 lines
17 KiB
Markdown
399 lines
17 KiB
Markdown
774- token = f" {needle} "
|
|
775- if token in filename_text or token in folder_text or token in source_text or token in excerpt_text:
|
|
776- matched_branches.add(branch)
|
|
777- break
|
|
778-
|
|
779- if len(matched_branches) == 1:
|
|
780- return next(iter(matched_branches))
|
|
781- return ""
|
|
782-
|
|
783-
|
|
784-def choose_candidates(rec: Record, destinations: list[str], limit: int = 14,
|
|
785- aliases: Optional[dict[str, list[str]]] = None,
|
|
786: allow_stay_outside_policy: bool = False) -> list[str]:
|
|
787- if not destinations:
|
|
788- return [rec.current_folder] if rec.current_folder else [""]
|
|
789-
|
|
790- # User-defined strong aliases route obvious domain shorthand before the LLM.
|
|
791- branch = _strong_route_branch(rec, aliases or {})
|
|
792- if branch:
|
|
793- routed = [d for d in destinations if d == branch or d.startswith(branch + "/")]
|
|
794- if routed:
|
|
795- destinations = routed
|
|
796-
|
|
797- text = f"{rec.filename} {rec.current_folder} {rec.source_context} {rec.excerpt[:400]}"
|
|
798- q = tokens(text)
|
|
--
|
|
821- for top in top_names:
|
|
822- if top in destinations and top not in chosen:
|
|
823- chosen.append(top)
|
|
824-
|
|
825- for _, folder in scored:
|
|
826- if folder not in chosen:
|
|
827- chosen.append(folder)
|
|
828- if len(chosen) >= limit:
|
|
829- break
|
|
830-
|
|
831- # For a real 01_INBOX we must always allow "leave it where it is" when the
|
|
832- # model cannot know enough. This can be a nested unpack folder as well.
|
|
833: if allow_stay_outside_policy and rec.current_folder and rec.current_folder not in chosen:
|
|
834- chosen.append(rec.current_folder)
|
|
835-
|
|
836: return chosen[:limit + (1 if allow_stay_outside_policy else 0)]
|
|
837-
|
|
838-
|
|
839-def scan(root: Path, target_root: Path, destinations: list[str],
|
|
840- aliases: Optional[dict[str, list[str]]] = None,
|
|
841: allow_stay_outside_policy: bool = False) -> list[Record]:
|
|
842- records = []
|
|
843- for p in sorted(root.rglob("*")):
|
|
844- if not p.is_file() or ignored(p, root):
|
|
845- continue
|
|
846- rel = str(p.relative_to(root))
|
|
847- try:
|
|
848: current = str(p.parent.relative_to(target_root))
|
|
849- except ValueError:
|
|
850- current = str(p.parent.relative_to(root))
|
|
851- if current == ".":
|
|
852- current = ""
|
|
853- # Keep provenance separate from destination logic. This matters when an
|
|
854- # incoming backup root itself carries useful ownership context (for
|
|
855- # example a Cafe-Bach backup) that would otherwise disappear when paths
|
|
856- # are made relative to the source root.
|
|
857- source_context = root.name
|
|
858- relative_parent = str(p.parent.relative_to(root))
|
|
859- if relative_parent != ".":
|
|
860- source_context = f"{root.name}/{relative_parent}"
|
|
--
|
|
870- current_folder=current,
|
|
871- date_guess=d,
|
|
872- date_source=source,
|
|
873- proposed_name=p.name,
|
|
874- destination=current,
|
|
875- status="archive" if archive else "pending",
|
|
876- content_status="archive" if archive else "not-read",
|
|
877- reason=("Archiv: entpacken, Inhalt sortieren, bei Bedarf sauber neu packen" if archive else ""),
|
|
878- source_context=source_context,
|
|
879- )
|
|
880- rec.candidates = choose_candidates(
|
|
881- rec, destinations, aliases=aliases,
|
|
882: allow_stay_outside_policy=allow_stay_outside_policy,
|
|
883- )
|
|
884- records.append(rec)
|
|
885- return records
|
|
886-
|
|
887-
|
|
888-# ---------------- LLM batching ----------------
|
|
889-
|
|
890-SYSTEM_PROMPT = """You are a conservative file-sorting assistant.
|
|
891-Your job is to save the human time, not create more decisions.
|
|
892-
|
|
893-Return one JSON ARRAY only, one object per input item, in the same order.
|
|
894-Each object must contain exactly:
|
|
--
|
|
997- if _result_confidence(result) < 0.85:
|
|
998- return True
|
|
999- if is_generic_filename(rec.filename):
|
|
1000- return True
|
|
1001- return False
|
|
1002-
|
|
1003-
|
|
1004-def llm_batches(
|
|
1005- records: list[Record],
|
|
1006- endpoint: str,
|
|
1007- model: str,
|
|
1008- batch_size: int = 1,
|
|
1009: fallback_model: str = "",
|
|
1010-):
|
|
1011- """Analyse exactly one file per LLM request.
|
|
1012-
|
|
1013- batch_size is retained for CLI/backward compatibility but intentionally
|
|
1014- ignored: one-file requests prevent cross-file influence and isolate failures.
|
|
1015-
|
|
1016- When a fallback model is configured, it is used only for a primary error,
|
|
1017- an invalid destination, confidence below 0.85, or a generic filename.
|
|
1018- If primary and fallback disagree on destination, the primary proposal is
|
|
1019- kept but confidence is capped at 0.69 so the GUI forces human review.
|
|
1020- """
|
|
1021- del batch_size
|
|
1022: fallback_model = fallback_model.strip()
|
|
1023-
|
|
1024- for rec in records:
|
|
1025- batch = [rec]
|
|
1026- primary = None
|
|
1027- primary_error = None
|
|
1028- try:
|
|
1029- primary = _call_llm_one(rec, endpoint, model)
|
|
1030- except Exception as exc:
|
|
1031- primary_error = str(exc)
|
|
1032-
|
|
1033- use_fallback = (
|
|
1034: bool(fallback_model)
|
|
1035: and fallback_model != model
|
|
1036- and (primary_error is not None or _needs_fallback(rec, primary))
|
|
1037- )
|
|
1038-
|
|
1039- if not use_fallback:
|
|
1040- if primary_error is not None:
|
|
1041- yield batch, None, primary_error
|
|
1042- else:
|
|
1043- yield batch, [primary], None
|
|
1044- continue
|
|
1045-
|
|
1046- try:
|
|
1047: fallback = _call_llm_one(rec, endpoint, fallback_model)
|
|
1048- except Exception as exc:
|
|
1049- fallback_error = str(exc)
|
|
1050- if primary is not None:
|
|
1051- # A usable primary answer is better than throwing it away merely
|
|
1052- # because the optional second opinion failed.
|
|
1053- yield batch, [primary], None
|
|
1054- else:
|
|
1055- yield batch, None, f"primary={primary_error}; fallback={fallback_error}"
|
|
1056- continue
|
|
1057-
|
|
1058- if primary is None:
|
|
1059- chosen = dict(fallback)
|
|
1060: chosen["reason"] = f"{fallback_model} fallback: {chosen.get('reason', '')}".strip()
|
|
1061- yield batch, [chosen], None
|
|
1062- continue
|
|
1063-
|
|
1064- primary_dest = str(primary.get("destination", "")).strip()
|
|
1065- fallback_dest = str(fallback.get("destination", "")).strip()
|
|
1066-
|
|
1067- if primary_dest == fallback_dest:
|
|
1068- # For the cases deliberately escalated, prefer the second opinion
|
|
1069- # when both models agree on the actual destination.
|
|
1070- chosen = dict(fallback)
|
|
1071: chosen["reason"] = f"{fallback_model} fallback: {chosen.get('reason', '')}".strip()
|
|
1072- yield batch, [chosen], None
|
|
1073- else:
|
|
1074- # Do not silently choose between disagreeing models. Keep the fast
|
|
1075- # primary proposal visible, but force explicit human review.
|
|
1076- chosen = dict(primary)
|
|
1077- chosen["confidence"] = min(_result_confidence(primary), 0.69)
|
|
1078- chosen["reason"] = (
|
|
1079- f"Model disagreement: {model}={primary_dest}; "
|
|
1080: f"{fallback_model}={fallback_dest}"
|
|
1081- )
|
|
1082- yield batch, [chosen], None
|
|
1083-
|
|
1084-
|
|
1085-def extract_json_array(text: str):
|
|
1086- text = text.strip()
|
|
1087- if text.startswith("```"):
|
|
1088- text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.I)
|
|
1089- text = re.sub(r"\s*```$", "", text)
|
|
1090- try:
|
|
1091- return json.loads(text)
|
|
1092- except json.JSONDecodeError:
|
|
--
|
|
1191- "enabled": False,
|
|
1192- "target_files": 0,
|
|
1193- "source_files": 0,
|
|
1194- "target_duplicates": 0,
|
|
1195- "source_duplicates": 0,
|
|
1196- "hash_errors": 0,
|
|
1197- "reason": "",
|
|
1198- }
|
|
1199-
|
|
1200- source_root = source_root.resolve()
|
|
1201- target_root = target_root.resolve()
|
|
1202-
|
|
1203: if source_root == target_root or source_root.is_relative_to(target_root):
|
|
1204- stats["reason"] = "source is the target or nested inside it; preserving normal V2 behaviour"
|
|
1205- return stats
|
|
1206- if target_root.is_relative_to(source_root):
|
|
1207- stats["reason"] = "target is nested inside source; unsafe layout for global dedupe"
|
|
1208- return stats
|
|
1209-
|
|
1210- stats["enabled"] = True
|
|
1211- target_by_hash: dict[str, Path] = {}
|
|
1212-
|
|
1213- for p in sorted(target_root.rglob("*")):
|
|
1214- if p.is_symlink() or not p.is_file() or ignored(p, target_root):
|
|
1215- continue
|
|
--
|
|
1227- # too: an archive that is already byte-identical in TARGET needs no LLM.
|
|
1228- try:
|
|
1229- digest = sha256_file(rec.path)
|
|
1230- except OSError:
|
|
1231- stats["hash_errors"] += 1
|
|
1232- continue
|
|
1233- stats["source_files"] += 1
|
|
1234-
|
|
1235- target_match = target_by_hash.get(digest)
|
|
1236- if target_match is not None:
|
|
1237- rec.duplicate_of = str(target_match)
|
|
1238- rec.status = "exact-duplicate"
|
|
1239: rec.collision = "EXACT → already in target"
|
|
1240- try:
|
|
1241: display = str(target_match.relative_to(target_root))
|
|
1242- except ValueError:
|
|
1243- display = str(target_match)
|
|
1244- rec.reason = f"SHA-256 match in target: {display}"
|
|
1245- stats["target_duplicates"] += 1
|
|
1246- continue
|
|
1247-
|
|
1248- first_source = source_seen.get(digest)
|
|
1249- if first_source is not None:
|
|
1250- rec.duplicate_of = str(first_source)
|
|
1251- rec.status = "exact-duplicate"
|
|
1252- rec.collision = "EXACT → duplicate incoming"
|
|
1253- try:
|
|
--
|
|
1447- if out:
|
|
1448- write_header = not log_path.exists()
|
|
1449- with log_path.open("a", newline="", encoding="utf-8") as f:
|
|
1450- writer = csv.DictWriter(f, fieldnames=out[0].keys())
|
|
1451- if write_header:
|
|
1452- writer.writeheader()
|
|
1453- writer.writerows(out)
|
|
1454- return out
|
|
1455-
|
|
1456-
|
|
1457-# ---------------- GUI ----------------
|
|
1458-
|
|
1459:def launch_gui(root: Path, target_root: Path, records: list[Record], destinations: list[str], endpoint: str, model: str, batch_size: int, fallback_model: str = "", aliases: Optional[dict[str, list[str]]] = None, inbox_mode: bool = False):
|
|
1460- import tkinter as tk
|
|
1461- from tkinter import ttk, messagebox
|
|
1462-
|
|
1463- class App:
|
|
1464- def __init__(self, win):
|
|
1465- self.win = win
|
|
1466- self.records = records
|
|
1467- self.destinations = destinations
|
|
1468- self.by_iid = {}
|
|
1469- self.q = queue.Queue()
|
|
1470- self.analysis_running = False
|
|
1471- self.active_editor = None
|
|
--
|
|
1535- rec.collision = ""
|
|
1536- plan(self.records)
|
|
1537- for iid in self.by_iid:
|
|
1538- self.refresh(iid)
|
|
1539-
|
|
1540- def rescan(self):
|
|
1541- if self.analysis_running:
|
|
1542- messagebox.showinfo("Busy", "Analysis is already running.")
|
|
1543- return
|
|
1544- old = {str(r.path): r for r in self.records}
|
|
1545- fresh = scan(
|
|
1546- root, target_root, self.destinations, aliases=aliases,
|
|
1547: allow_stay_outside_policy=inbox_mode,
|
|
1548- )
|
|
1549- merged = []
|
|
1550- new_count = 0
|
|
1551- for rec in fresh:
|
|
1552- previous = old.get(str(rec.path))
|
|
1553- if previous is not None:
|
|
1554- merged.append(previous)
|
|
1555- else:
|
|
1556- merged.append(rec)
|
|
1557- new_count += 1
|
|
1558- self.records = merged
|
|
1559- global_duplicate_prepass(self.records, root, target_root)
|
|
--
|
|
1589- )
|
|
1590- else:
|
|
1591- messagebox.showinfo("Select files", "Select one or more rows first.")
|
|
1592- return
|
|
1593- self.analysis_running = True
|
|
1594- self.status.config(text=f"Analysing {len(chosen)} file(s), one request per file…")
|
|
1595-
|
|
1596- def worker():
|
|
1597- for rec in chosen:
|
|
1598- if not rec.excerpt:
|
|
1599- rec.excerpt, rec.content_status = extract_excerpt(rec.path)
|
|
1600- rec.candidates = choose_candidates(rec, self.destinations, aliases=aliases)
|
|
1601: for batch, results, error in llm_batches(chosen, endpoint, model, batch_size, fallback_model=fallback_model):
|
|
1602- self.q.put((batch, results, error))
|
|
1603- self.q.put((None, None, "__DONE__"))
|
|
1604-
|
|
1605- threading.Thread(target=worker, daemon=True).start()
|
|
1606- self.win.after(100, self.poll)
|
|
1607-
|
|
1608- def poll(self):
|
|
1609- try:
|
|
1610- while True:
|
|
1611- batch, results, error = self.q.get_nowait()
|
|
1612- if error == "__DONE__":
|
|
1613- self.analysis_running = False
|
|
--
|
|
1729- "No existing file will be overwritten. Exact duplicates are only "
|
|
1730- "removed after a full-file SHA-256 match."
|
|
1731- )
|
|
1732- if not messagebox.askyesno("Apply resolved batch?", summary):
|
|
1733- return
|
|
1734- result = apply_confirmed(self.records, root / LOG_FILENAME)
|
|
1735- moved = sum(1 for x in result if x["result"] in ("moved", "moved as variant"))
|
|
1736- removed = sum(1 for x in result if x["result"] == "removed exact duplicate")
|
|
1737- skipped = sum(1 for x in result if x["result"].startswith("SKIPPED"))
|
|
1738- if inbox_mode:
|
|
1739- self.records = scan(
|
|
1740- root, target_root, self.destinations, aliases=aliases,
|
|
1741: allow_stay_outside_policy=True,
|
|
1742- )
|
|
1743- self._populate()
|
|
1744- messagebox.showinfo(
|
|
1745- "Done",
|
|
1746- f"Moved/renamed: {moved}\nRemoved exact duplicates: {removed}\nSkipped: {skipped}\n"
|
|
1747- f"Remaining in Inbox: {len(self.records) if inbox_mode else 'n/a'}\nLog: {root / LOG_FILENAME}"
|
|
1748- )
|
|
1749-
|
|
1750- def sort(self, column, reverse):
|
|
1751- rows = [(self.tree.set(iid, column), iid) for iid in self.tree.get_children("")]
|
|
1752- rows.sort(key=lambda x: x[0].lower(), reverse=reverse)
|
|
1753- for pos, (_, iid) in enumerate(rows):
|
|
--
|
|
1813- target_root = Path(args.target_root).expanduser().resolve()
|
|
1814- elif inbox_mode:
|
|
1815- target_root = root.parent
|
|
1816- else:
|
|
1817- target_root = root
|
|
1818- if not target_root.is_dir():
|
|
1819- raise SystemExit(f"Not a target directory: {target_root}")
|
|
1820-
|
|
1821- destinations = allowed_destinations(root, policy, args.profile)
|
|
1822- aliases = policy.get("_strong_aliases", {}).get(profile, {})
|
|
1823- records = scan(
|
|
1824- root, target_root, destinations, aliases=aliases,
|
|
1825: allow_stay_outside_policy=inbox_mode,
|
|
1826- )
|
|
1827- dedupe = global_duplicate_prepass(records, root, target_root)
|
|
1828-
|
|
1829- print(f"Root: {root}")
|
|
1830- print(f"Target root: {target_root}")
|
|
1831- print(f"Policy profile: {profile}")
|
|
1832- print(f"Files: {len(records)}")
|
|
1833- print(f"Archives: {sum(r.status == 'archive' for r in records)}")
|
|
1834- if dedupe["enabled"]:
|
|
1835- print(
|
|
1836: "Global SHA-256 pre-pass: "
|
|
1837- f"target files={dedupe['target_files']}, source files={dedupe['source_files']}, "
|
|
1838: f"already in target={dedupe['target_duplicates']}, "
|
|
1839- f"duplicate incoming={dedupe['source_duplicates']}, hash errors={dedupe['hash_errors']}"
|
|
1840- )
|
|
1841- else:
|
|
1842: print(f"Global SHA-256 pre-pass: disabled ({dedupe['reason']})")
|
|
1843- print(f"Allowed destinations: {len(destinations)}")
|
|
1844- print(f"Strong route aliases: {sum(len(v) for v in aliases.values())} across {len(aliases)} branches")
|
|
1845- print(f"Date from filename: {sum(r.date_source == 'filename' for r in records)}")
|
|
1846- print(f"Date from dated parent folder: {sum(r.date_source == 'folder' for r in records)}")
|
|
1847- print(f"Date from mtime fallback: {sum(r.date_source == 'mtime' for r in records)}")
|
|
1848-
|
|
1849: fallback_model = "" if args.no_fallback else args.fallback_model.strip()
|
|
1850- print(f"Primary LLM model: {args.model}")
|
|
1851: print(f"Fallback LLM model: {fallback_model or 'disabled'}")
|
|
1852-
|
|
1853- if args.scan_only:
|
|
1854- for r in records[:25]:
|
|
1855- duplicate = f" -> {r.duplicate_of}" if r.duplicate_of else ""
|
|
1856- print(f"{r.status:15} {r.date_guess} [{r.date_source:8}] {r.rel_path}{duplicate}")
|
|
1857- return
|
|
1858-
|
|
1859: launch_gui(root, target_root, records, destinations, args.endpoint, args.model, max(1, args.batch_size), fallback_model=fallback_model, aliases=aliases, inbox_mode=inbox_mode)
|
|
1860-
|
|
1861-
|
|
1862-if __name__ == "__main__":
|
|
1863- main()
|