import { ingestPages, type IngestReport } from "./document-ingestion"; import type { DocumentFixture } from "../../content/approach-document-fixture"; export interface PageReview { page: number; classification: DocumentFixture["classification"]; readings: readonly string[]; issues: Array<"disagreement" | "identifier" | "notation" | "missing">; state: "ready" | "review" | "missing"; } export interface DocumentRun { report: IngestReport; pages: PageReview[] } export interface ReviewDecision { text: string; reviewer: "visitor" } export type ReviewDecisions = Partial>; /** Compare prepared passes; disagreement is not an OCR confidence estimate. */ export function inspectReadings(readings: readonly string[]): PageReview["issues"] { if (!readings.length || readings.every(text => !text.trim())) return ["missing"]; const issues: PageReview["issues"] = []; const normalized = readings.map(text => text.replace(/\s+/g, " ").trim()); if (new Set(normalized).size > 1) issues.push("disagreement"); if (normalized.some(text => !/\bST-\d{4}\b/.test(text))) issues.push("identifier"); // This narrow rule flags a known ambiguity; it does not validate mathematics. if (normalized.some(text => /\bx\d/.test(text)) && normalized.some(text => /x[²³^]/.test(text))) issues.push("notation"); return issues; } export async function processDocument(pages: readonly DocumentFixture[], ocrUnavailable = false): Promise { const report = await ingestPages("WORKSHOP", pages.map(page => ({ page: page.page, embeddedText: page.embeddedText, imagePath: page.embeddedText ? undefined : `sample-page-${page.page}`, })), async path => { const page = pages.find(item => `sample-page-${item.page}` === path); if (ocrUnavailable || !page?.readings) throw new Error("No sample OCR reading available"); return page.readings[0]; }, 160); return { report, pages: pages.map(page => { const extracted = report.chunks.filter(chunk => chunk.page === page.page); const readings = page.embeddedText ? [page.embeddedText] : extracted.length ? [...(page.readings ?? [])] : []; const issues = inspectReadings(readings); return { page: page.page, classification: page.classification, readings, issues, state: !extracted.length ? "missing" : issues.length ? "review" : "ready" }; }) }; } /** Review decisions remain separate from immutable extraction records. */ export function buildEvidencePacket(run: DocumentRun, decisions: ReviewDecisions, query: string, budget = 600) { if (!Number.isInteger(budget) || budget < 1) throw new Error("Invalid context budget"); const terms = [...new Set(query.toLowerCase().match(/[a-z0-9]+/g) ?? [])]; const candidates = run.pages.flatMap(page => { if (page.state === "missing" || (page.state === "review" && !decisions[page.page])) return []; const text = decisions[page.page]?.text ?? page.readings[0]; if (!text?.trim()) return []; const lower = text.toLowerCase(); const score = terms.filter(term => lower.includes(term)).length; if (!score) return []; return [{ sourceId: `${run.report.documentId}:p${page.page}`, page: page.page, classification: page.classification, text, score, reviewed: Boolean(decisions[page.page]), extractedTextHash: run.report.extractedTextHash, chunkIds: run.report.chunks.filter(chunk => chunk.page === page.page).map(chunk => chunk.id) }]; }).sort((a,b) => b.score-a.score || a.page-b.page); let used = 0; const records = candidates.filter(record => { // Budget the complete serialized record, including its source metadata. const size = JSON.stringify(record).length + 1; if (used + size > budget) return false; used += size; return true; }); return { records, used, budget, eligibleMatches: candidates.length }; }