10 KiB
899:SYSTEM_PROMPT = """You are a conservative file-sorting assistant. 900-Your job is to save the human time, not create more decisions. 901- 902-Return one JSON ARRAY only, one object per input item, in the same order. 903-Each object must contain exactly: 904-id, destination, topic, subtopic, document_type, detail, title, subtitle, version, confidence, reason 905- 906-Rules: 907-- destination MUST be one of that item's candidates. Never invent a folder. 908-- In 01_INBOX mode, choose the stable owner/domain first; use 99_ARCHIV only for clearly historical/reference material, not merely because a file is old. 909-- If uncertain, keep the current folder when it is a candidate, lower confidence, and say why. 910-- Do NOT write the final filename. Extract semantic naming fields; Python will build the filename consistently. 911-- title: concise German document label for the filename, e.g. Anschreiben, Inventur, Rechnung, Getränkeliste, Checkliste. 912-- subtitle: concise distinguishing subject/entity/event/year, e.g. Phönix-Sauna, 2023, CB2428-Aidshilfe-Köln; empty if unnecessary. 913-- title/subtitle MUST NOT contain the date, destination/prefix, extension, or version and MUST NOT repeat each other. 914-- Prefer normal readable German spelling and capitalization in title/subtitle; do not slugify or translate into English. 915-- topic: stable broad context/owner, e.g. cafe-bach, checkpoint, csd, sp5. 916-- subtopic: distinctive event/project/subject, e.g. bach-voices, golden-gays-versammlung, chemsex. 917-- document_type: what the document is, e.g. ablaufplan, protokoll, rechnung, handout. 918-- detail: optional useful distinguishing detail; empty string if unnecessary. 919-- version: explicit version only if supported by filename/content; otherwise empty string. 920-- Use only as many semantic parts as useful; never invent filler. 921-- The filename is NOT the only source of truth. Use the excerpt to recover the actual project, event, organisation, or subject when clearly stated. 922-- source_context is provenance from the incoming source root/folder. Use it as supporting evidence for the primary owner/work area; generic backup labels are not destinations by themselves. 923-- If generic_name=true (for example "Ablaufplan", "Protokoll", "Präsentation", "Rechnung"), DO NOT merely preserve/paraphrase the generic filename. Add the distinctive project/event/subject from the content when it is clear. 924-- For destination, prefer the PRIMARY OWNER/WORK AREA of the document. A mentioned event/topic can be secondary context. Example principle: an inventory/accounting document owned by one work area does not switch branches merely because it mentions a campaign/event. 925-- If several work-area aliases occur, do not assume the longest alias is primary; use filename structure + content semantics to decide and lower confidence if genuinely ambiguous. 926-- Use an informative original filename as evidence for title/subtitle, but do not preserve its whole formatting; Python will normalize the final name. 927-- Use the supplied date_guess; do not invent a different date from vague content. 928-- If date_source is mtime, treat the date as unverified fallback metadata. Do not invent a different date. 929-- Correct obvious spelling/transliteration problems when safe. 930-- confidence is 0.0 to 1.0. Use >=0.85 only for genuinely clear cases. 931-- reason: max 12 words, factual and concise. 932-""" 933- 934- 935-def _model_generation_limits(model: str) -> tuple[int, int]: 936- """Return a generous output-token ceiling and HTTP timeout for one file. 937- 938- Reasoning models spend part of the completion budget before emitting final 939- JSON. These are ceilings, not targets: models can stop much earlier. 940- """ 941- name = model.strip().lower() 942- if "debug" in name: 943- return 6000, 300 944- if "architect" in name: 945- return 2500, 180 946- if "daily" in name: 947- return 1000, 120 948- return 2000, 180 949- 950- 951-def _result_confidence(result: Optional[dict]) -> float: 952- if not isinstance(result, dict): 953- return 0.0 954- try: 955- return max(0.0, min(1.0, float(result.get("confidence", 0)))) 956- except (TypeError, ValueError): 957- return 0.0 958- 959- 960-def _call_llm_one(rec: Record, endpoint: str, model: str) -> dict: 961- item = { 962- "id": 0, 963- "filename": rec.filename, 964- "current_folder": rec.current_folder, 965- "source_context": rec.source_context, 966- "date_guess": rec.date_guess, 967- "date_source": rec.date_source, 968- "generic_name": is_generic_filename(rec.filename), 969- "content_status": rec.content_status, 970- "candidates": rec.candidates, 971- "excerpt": rec.excerpt[:MAX_EXCERPT], 972- } 973- prompt = SYSTEM_PROMPT + "\nINPUT:\n" + json.dumps([item], ensure_ascii=False) 974- max_tokens, timeout = _model_generation_limits(model) 975- payload = { 976- "model": model, 977- "messages": [{"role": "user", "content": prompt}], 978- "temperature": 0.0, 979- "max_tokens": max_tokens, 980- } 981- req = urllib.request.Request( 982- endpoint, 983- data=json.dumps(payload).encode("utf-8"), 984- headers={"Content-Type": "application/json"}, 985- method="POST", 986- ) 987- with urllib.request.urlopen(req, timeout=timeout) as resp: 988- data = json.loads(resp.read().decode("utf-8")) 989- choice = data["choices"][0] 990- text = choice.get("message", {}).get("content") or "" 991- parsed = extract_json_array(text) 992- if not isinstance(parsed, list) or not parsed or not isinstance(parsed[0], dict): 993- raise ValueError("model did not return a non-empty JSON array") 994- result = dict(parsed[0]) 995- result["id"] = 0 996- return result 997- 998- 999-def _needs_fallback(rec: Record, result: Optional[dict]) -> bool: 1000- """Escalate only uncertain/fragile cases; do not pay Architect cost always.""" 1001- if not isinstance(result, dict): 1002- return True 1003- dest = str(result.get("destination", "")).strip() 1004- if dest not in rec.candidates: 1005- return True 1006- if _result_confidence(result) < 0.85: 1007- return True 1008- if is_generic_filename(rec.filename): 1009- return True 1010- return False 1011- 1012- 1013-def llm_batches( 1014- records: list[Record], 1015- endpoint: str, 1016- model: str, 1017- batch_size: int = 1, 1018- fallback_model: str = "", 1019-): 960:def _call_llm_one(rec: Record, endpoint: str, model: str) -> dict: 961- item = { 962- "id": 0, 963- "filename": rec.filename, 964- "current_folder": rec.current_folder, 965- "source_context": rec.source_context, 966- "date_guess": rec.date_guess, 967- "date_source": rec.date_source, 968- "generic_name": is_generic_filename(rec.filename), 969- "content_status": rec.content_status, 970- "candidates": rec.candidates, 971- "excerpt": rec.excerpt[:MAX_EXCERPT], 972- } 973- prompt = SYSTEM_PROMPT + "\nINPUT:\n" + json.dumps([item], ensure_ascii=False) 974- max_tokens, timeout = _model_generation_limits(model) 975- payload = { 976- "model": model, 977- "messages": [{"role": "user", "content": prompt}], 978- "temperature": 0.0, 979- "max_tokens": max_tokens, 980- } 981- req = urllib.request.Request( 982- endpoint, 983- data=json.dumps(payload).encode("utf-8"), 984- headers={"Content-Type": "application/json"}, 985- method="POST", 986- ) 987- with urllib.request.urlopen(req, timeout=timeout) as resp: 988- data = json.loads(resp.read().decode("utf-8")) 989- choice = data["choices"][0] 990- text = choice.get("message", {}).get("content") or "" 991- parsed = extract_json_array(text) 992- if not isinstance(parsed, list) or not parsed or not isinstance(parsed[0], dict): 993- raise ValueError("model did not return a non-empty JSON array") 994- result = dict(parsed[0]) 995- result["id"] = 0 996- return result 997- 998- 999-def _needs_fallback(rec: Record, result: Optional[dict]) -> bool: 1000- """Escalate only uncertain/fragile cases; do not pay Architect cost always.""" 1001- if not isinstance(result, dict): 1002- return True 1003- dest = str(result.get("destination", "")).strip() 1004- if dest not in rec.candidates: 1005- return True 1006- if _result_confidence(result) < 0.85: 1007- return True 1008- if is_generic_filename(rec.filename): 1009- return True 1010- return False 1011- 1012- 1013-def llm_batches( 1014- records: list[Record], 1015- endpoint: str, 1016- model: str, 1017- batch_size: int = 1, 1018- fallback_model: str = "", 1019-): 1020- """Analyse exactly one file per LLM request. 1021- 1022- batch_size is retained for CLI/backward compatibility but intentionally 1023- ignored: one-file requests prevent cross-file influence and isolate failures. 1024- 1025- When a fallback model is configured, it is used only for a primary error, 1026- an invalid destination, confidence below 0.85, or a generic filename. 1027- If primary and fallback disagree on destination, the primary proposal is 1028- kept but confidence is capped at 0.69 so the GUI forces human review. 1029- """ 1030- del batch_size 1031- fallback_model = fallback_model.strip() 1032- 1033- for rec in records: 1034- batch = [rec] 1035- primary = None 1036- primary_error = None 1037- try: 1038- primary = _call_llm_one(rec, endpoint, model) 1039- except Exception as exc: 1040- primary_error = str(exc) 1041- 1042- use_fallback = ( 1043- bool(fallback_model) 1044- and fallback_model != model 1045- and (primary_error is not None or _needs_fallback(rec, primary)) 1046- ) 1047- 1048- if not use_fallback: 1049- if primary_error is not None: 1050- yield batch, None, primary_error 1051- else: 1052- yield batch, [primary], None 1053- continue 1054- 1055- try: 1056- fallback = _call_llm_one(rec, endpoint, fallback_model) 1057- except Exception as exc: 1058- fallback_error = str(exc) 1059- if primary is not None: 1060- # A usable primary answer is better than throwing it away merely