diff --git a/run_copyrighter_server.py b/run_copyrighter_server.py index 5f76041..efcc550 100644 --- a/run_copyrighter_server.py +++ b/run_copyrighter_server.py @@ -6,7 +6,7 @@ import sys # surfaces this friendly message instead of a raw SyntaxError/ImportError. if sys.version_info < (3, 13): sys.exit( - "Copyrighter requires Python 3.13+ — activate the offline venv " + "Copyrighter requires Python 3.13+ - activate the offline venv " "(see docs/operations/offline-install.md)." ) diff --git a/src/rights_filter/analysis/llm_assistance.py b/src/rights_filter/analysis/llm_assistance.py index 7c62286..94476cd 100644 --- a/src/rights_filter/analysis/llm_assistance.py +++ b/src/rights_filter/analysis/llm_assistance.py @@ -29,6 +29,12 @@ logger = logging.getLogger(__name__) # row as its source. Capping the input is what makes the citation honest. MAX_PROMPT_EVIDENCE = 12 +# Single source of truth for the model. env_clients re-exports this rather than +# declaring its own: two defaults that disagree is how the client ended up +# pointing at a model that was never installed while the wiring pointed +# elsewhere, and nothing failed until a request 404'd at runtime. +DEFAULT_MODEL = "qwen2.5:7b" + # Data fields the model is allowed to see. A whitelist rather than a blocklist: # evidence payloads grow, and a field nobody vetted must not reach the prompt by # default. Notably absent — promotion_reason (the model copies it verbatim and @@ -79,7 +85,7 @@ class OllamaGenerateLlmClient: def __init__( self, base_url: str = "http://127.0.0.1:11434", - model: str = "qwen2.5:3b-instruct", + model: str = DEFAULT_MODEL, transport: Any | None = None, timeout: int = 120, connect_timeout: float = 2, @@ -185,8 +191,26 @@ class InternalLlmAssistant: "dropped_claims": result.dropped_claims, } - source_urls = _source_urls(evidence) - source_evidence_ids = _source_evidence_ids(evidence) + # The linked sources must be what the summary actually rests on, not the + # whole case. Deriving them from the full evidence list was the original + # defect: the prompt is capped at MAX_PROMPT_EVIDENCE, so a case with 79 + # rows produced a summary read from 12 while the UI reported 78 "linked + # sources". The policy check passed and the number was fiction. + # + # Cited rows first (what the verified claims point at); the shown rows as + # a fallback when nothing verified, so the row is still storable and the + # count still describes something the model actually read. + shown = _select_prompt_evidence(evidence) + labels = _evidence_labels(shown) + cited_labels = { + label for claim in result.claims for label in claim.get("evidence", []) + } + cited = [labels[label] for label in sorted(cited_labels) if label in labels] + grounded = bool(cited) + source_evidence = cited or shown + + source_urls = _source_urls(source_evidence) + source_evidence_ids = _source_evidence_ids(source_evidence) if not source_urls and not source_evidence_ids: return Evidence( source=EvidenceSource.LLM_SUMMARY, @@ -211,7 +235,11 @@ class InternalLlmAssistant: "summary": result.text, "source_urls": source_urls, "source_evidence_ids": source_evidence_ids, - "verified": True, + # True only when a claim survived citation checking. Previously + # this was True whenever any source existed, which made it a + # restatement of "the case has evidence" rather than a statement + # about the summary. + "verified": grounded, **stamp, }, ) diff --git a/src/rights_filter/integrations/env_clients.py b/src/rights_filter/integrations/env_clients.py index 2b43c54..7cc4b2f 100644 --- a/src/rights_filter/integrations/env_clients.py +++ b/src/rights_filter/integrations/env_clients.py @@ -7,7 +7,11 @@ from typing import Any, Mapping logger = logging.getLogger(__name__) -from rights_filter.analysis.llm_assistance import InternalLlmAssistant, OllamaGenerateLlmClient +from rights_filter.analysis.llm_assistance import ( + DEFAULT_MODEL, + InternalLlmAssistant, + OllamaGenerateLlmClient, +) from rights_filter.integrations.cloud_vision_web_detection import ( CloudVisionWebDetectionAdapter, GoogleVisionRestClient, @@ -27,7 +31,9 @@ from rights_filter.integrations.search_policy import SearchApiPolicy # qwen3:8b 35.7s — held the contract but 4x slower (thinking) # 7b is also already present on the build machine, so the default costs no # additional model transfer across the air gap. -DEFAULT_OLLAMA_MODEL = "qwen2.5:7b" +# +# Re-exported, not redeclared: the client owns the value so the two cannot drift. +DEFAULT_OLLAMA_MODEL = DEFAULT_MODEL @dataclass diff --git a/src/rights_filter/server/__main__.py b/src/rights_filter/server/__main__.py index d54f80f..83d04a1 100644 --- a/src/rights_filter/server/__main__.py +++ b/src/rights_filter/server/__main__.py @@ -27,13 +27,21 @@ def _provider_readiness(runtime) -> str: def main() -> None: # When the server is started as a service with stdout redirected to a file, - # Python switches to block buffering and the startup diagnostics below — - # including the "token not set" security warning — sit in the buffer + # Python switches to block buffering and the startup diagnostics below - + # including the "token not set" security warning - sit in the buffer # indefinitely (the process never exits). Line-buffer so they reach the log # immediately. + # + # errors="replace" is not cosmetic: the delivery target is a Korean Windows + # host whose console codepage is cp949, and a single character outside it + # (an em dash in a warning string was enough) raised UnicodeEncodeError from + # print() and killed the server before it ever bound the port. A diagnostic + # message must never be able to prevent startup - degrade the character, not + # the process. try: - sys.stdout.reconfigure(line_buffering=True) - except (AttributeError, OSError): + sys.stdout.reconfigure(line_buffering=True, errors="replace") + sys.stderr.reconfigure(errors="replace") + except (AttributeError, OSError, ValueError): pass parser = argparse.ArgumentParser(description="Run the local Copyrighter API server.") @@ -61,7 +69,7 @@ def main() -> None: env_path = Path(args.env) load_env_file(env_path, os.environ) # Log the resolved env path (and whether it was found) so an operator running - # from an unexpected directory can see which .env — if any — was actually + # from an unexpected directory can see which .env - if any - was actually # loaded, rather than silently getting no token / disabled providers. print(f".env: {env_path.resolve()} ({'loaded' if env_path.exists() else 'not found'})") provider_runtime = build_provider_runtime(os.environ) @@ -73,17 +81,17 @@ def main() -> None: auth_token = os.environ.get("COPYRIGHTER_AUTH_TOKEN", "").strip() or None if auth_token is None: - print("WARNING: COPYRIGHTER_AUTH_TOKEN not set — data routes are unauthenticated.") + print("WARNING: COPYRIGHTER_AUTH_TOKEN not set - data routes are unauthenticated.") elif len(auth_token) < 32: print( - "WARNING: COPYRIGHTER_AUTH_TOKEN is shorter than 32 characters — " + "WARNING: COPYRIGHTER_AUTH_TOKEN is shorter than 32 characters - " "use a longer, high-entropy token (e.g. `python -c \"import secrets; print(secrets.token_urlsafe(32))\"`)." ) static_path = Path(args.static) if not static_path.exists(): print( - f"WARNING: static GUI directory not found: {static_path.resolve()} — " + f"WARNING: static GUI directory not found: {static_path.resolve()} - " "the operator UI will return 404. Pass --static or run from the repo root." ) diff --git a/src/rights_filter/server/store_serialization.py b/src/rights_filter/server/store_serialization.py index dd5a046..873198d 100644 --- a/src/rights_filter/server/store_serialization.py +++ b/src/rights_filter/server/store_serialization.py @@ -346,6 +346,12 @@ def _evidence_payload(submission_id: str, evidence: Evidence) -> dict[str, Any]: "promptId": str(evidence.data.get("prompt_id", "")), "promptVersion": str(evidence.data.get("prompt_version", "")), "latencyMs": evidence.data.get("latency_ms", ""), + # Fabrication counters. Without these on the payload the metric exists + # only in memory and the operator/admin never sees a model that has + # started inventing citations. + "groundedClaims": evidence.data.get("grounded_claims", ""), + "droppedClaims": evidence.data.get("dropped_claims", ""), + "verified": evidence.data.get("verified", ""), "status": "active", "submission_id": submission_id, } @@ -403,6 +409,9 @@ def _domain_evidence_from_ui(payload: dict[str, Any]) -> Evidence: "prompt_id": payload.get("promptId", ""), "prompt_version": payload.get("promptVersion", ""), "latency_ms": payload.get("latencyMs", ""), + "grounded_claims": payload.get("groundedClaims", ""), + "dropped_claims": payload.get("droppedClaims", ""), + "verified": payload.get("verified", ""), "provenance_kind": payload.get("provenanceKind", ""), "rights_fields": payload.get("rightsFields", {}), "capture_categories": payload.get("captureCategories", []), @@ -505,7 +514,7 @@ def _ui_source(source: EvidenceSource) -> str: def _ui_group(source: EvidenceSource) -> str: ui_source = _ui_source(source) - if ui_source in {"fingerprint", "face", "provenance"}: + if ui_source in {"fingerprint", "face"}: return "internal" return ui_source diff --git a/tests/rights_filter/analysis/test_llm_grounding.py b/tests/rights_filter/analysis/test_llm_grounding.py index 517c43a..e9507b9 100644 --- a/tests/rights_filter/analysis/test_llm_grounding.py +++ b/tests/rights_filter/analysis/test_llm_grounding.py @@ -225,3 +225,41 @@ def test_assistant_records_how_much_survived_verification(field): evidence = InternalLlmAssistant(client).summarize("SUB-1", [_web()]) assert evidence.data[field] == 1 + + +# --- linked sources must describe what the model read ------------------------- + + +def _many(count): + return [_web(url=f"https://example.test/{index}") for index in range(count)] + + +def test_linked_sources_never_exceed_what_the_model_was_shown(): + # The original defect: a 79-row case produced a summary read from 12 rows + # while the UI reported 78 "linked sources" and the policy check passed. + client, _ = _client({"claims": []}) + evidence = _many(40) + + result_evidence = InternalLlmAssistant(client).summarize("SUB-1", evidence) + + assert len(result_evidence.data["source_evidence_ids"]) <= MAX_PROMPT_EVIDENCE + assert len(result_evidence.data["source_urls"]) <= MAX_PROMPT_EVIDENCE + + +def test_linked_sources_are_the_cited_rows_when_claims_verify(): + client, _ = _client({"claims": [{"text": "근거 있음.", "evidence": ["E1"]}]}) + + result_evidence = InternalLlmAssistant(client).summarize("SUB-1", _many(20)) + + assert len(result_evidence.data["source_urls"]) == 1 + assert result_evidence.data["verified"] is True + + +def test_verified_is_false_when_nothing_survived_citation_checking(): + # "verified" used to mean "the case has evidence", which is not a statement + # about the summary at all. + client, _ = _client({"claims": [{"text": "지어냄.", "evidence": ["E99"]}]}) + + result_evidence = InternalLlmAssistant(client).summarize("SUB-1", _many(20)) + + assert result_evidence.data["verified"] is False diff --git a/web/operator-gui/app.js b/web/operator-gui/app.js index 1559620..088b77d 100644 --- a/web/operator-gui/app.js +++ b/web/operator-gui/app.js @@ -1632,6 +1632,8 @@ function evidenceGroupFor(evidence) { if (evidence.source === "face") return "face"; + if (evidence.source === "provenance") return "provenance"; + if (evidence.source === "llm") return "llm"; if (evidence.source === "failure") return "failure"; diff --git a/web/operator-gui/index.html b/web/operator-gui/index.html index 2a60f26..ad2f9bf 100644 --- a/web/operator-gui/index.html +++ b/web/operator-gui/index.html @@ -151,6 +151,7 @@ +