MVP M8: background detection — assumptions, risks, inconsistencies
The editor now runs Socrates' three Phase-0-validated detection prompts against the live model, persists the findings, and surfaces them in a new FindingsPanel beside the IssuesPanel. Click any finding to focus its linked element across rail + diagram. Re-detect after model edits to refresh against the new state. apps/web/lib/llm/prompts/socrates - detect-assumptions.md / detect-risks.md / detect-inconsistencies.md promoted verbatim from phase-0 (Phase 0 corpus validated them 10/10). apps/web/lib/llm/detect.ts - Three sequential detection passes (parallel was OOM-prone on 4B local models — Phase 0 lesson). Each pass uses the Phase 0 JSON schema with jsonObjectMode fallback + chatJSON repair-retry. Fail-soft per pass: one busted pass returns [] rather than blowing up the whole detect. - post-validate strips hallucinated element refs (drops findings whose refs ALL fail to resolve; keeps findings with zero refs since some inconsistencies are genuinely about absences). apps/web/prisma/schema.prisma - Finding table: kind / text / linkedElementIds (JSON) / confidence / severity / validationCode / status / modelVersion / provider / model. - ResearchFinding table reserved for the Tavily integration that comes next — schema in place so we don't have to migrate again. apps/web/lib/db/repo.ts - listOpenFindings(projectId), replaceFindings(...) — replaceFindings wipes prior open findings in a transaction and writes the new set so re-detect doesn't accumulate stale findings. apps/web/app/api/projects/[projectId]/findings/route.ts - GET returns persisted open findings. - POST runs detect, persists, returns findings + meta (provider, model, durationMs, strippedRefs, droppedFindings). apps/web/components/editor/FindingsPanel.tsx - New panel, anchored bottom-right just left of IssuesPanel. Shows count summary (asm / risk / inc), detect / re-detect button, list grouped by kind (inconsistencies first, then risks, then assumptions), per-finding glyph + tag + severity + confidence + linked refs. - Click a finding row → focus its first linked element via the same setFocusBlockId path the rail and IssuesPanel already use. - "stale" indicator when the model version has advanced past the one the findings were detected against. EditorShell wires version + projectId through to FindingsPanel. Smoke-tested end-to-end: 12 findings returned (5 asm / 4 risk / 3 inc), 0 hallucinated refs stripped, ~37s on local gemma-4-e4b. Sample assumption "students are willing to engage with an AI tutor that is programmed to refuse providing complete solutions" — specific to Aristotle's refusal_policy, not a generic startup truism. Deferred to follow-ups: inline rail/diagram badges from findings, auto-detect-on-save, Tavily research, experiment modal.
This commit is contained in:
301
apps/web/lib/llm/detect.ts
Normal file
301
apps/web/lib/llm/detect.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
// Background detection — runs three Phase-0-validated detection prompts
|
||||
// against the current model and returns structured findings.
|
||||
//
|
||||
// Sequential not parallel: small local models share a KV-cache budget; three
|
||||
// concurrent calls can OOM. Each call is independent so total wall time
|
||||
// roughly equals 3× per-call time, but stability is much higher.
|
||||
//
|
||||
// Post-validation strips hallucinated element ids — the same defensive step
|
||||
// that rescued ~5% of phase-0 detection runs.
|
||||
|
||||
import "server-only";
|
||||
import { defaultGateway, chatJSON, type Message } from "./gateway";
|
||||
import { loadPrompt } from "./prompts";
|
||||
import type { SysMLModel } from "../sysml/model";
|
||||
|
||||
// ─── Output types ────────────────────────────────────────────────────────
|
||||
|
||||
export type FindingKind = "assumption" | "risk" | "inconsistency";
|
||||
|
||||
export interface Finding {
|
||||
kind: FindingKind;
|
||||
text: string;
|
||||
linkedElementIds: string[];
|
||||
confidence: number;
|
||||
severity?: "low" | "medium" | "high";
|
||||
validationCode?: string;
|
||||
}
|
||||
|
||||
export interface DetectResult {
|
||||
findings: Finding[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
strippedRefs: number;
|
||||
droppedFindings: number;
|
||||
provider: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
// ─── Per-pass JSON schemas (oneOf-by-shape; small models cope) ──────────
|
||||
|
||||
const baseProps = {
|
||||
text: { type: "string", minLength: 1 },
|
||||
linkedElementIds: { type: "array", items: { type: "string" } },
|
||||
confidence: { type: "number", minimum: 0, maximum: 1 },
|
||||
};
|
||||
|
||||
const assumptionsJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence"],
|
||||
properties: baseProps,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const risksJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence", "severity"],
|
||||
properties: {
|
||||
...baseProps,
|
||||
severity: { type: "string", enum: ["low", "medium", "high"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const inconsistenciesJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence"],
|
||||
properties: {
|
||||
...baseProps,
|
||||
validationCode: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
interface RawFinding {
|
||||
text?: string;
|
||||
linkedElementIds?: string[];
|
||||
confidence?: number;
|
||||
severity?: "low" | "medium" | "high";
|
||||
validationCode?: string;
|
||||
}
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export async function detectFindings(model: SysMLModel): Promise<DetectResult> {
|
||||
const gateway = defaultGateway();
|
||||
const start = Date.now();
|
||||
let totalIn = 0;
|
||||
let totalOut = 0;
|
||||
|
||||
const userPayload = buildUserPayload(model);
|
||||
|
||||
const assumptions = await runPass(gateway, "detect-assumptions", userPayload, assumptionsJsonSchema);
|
||||
totalIn += assumptions.inputTokens;
|
||||
totalOut += assumptions.outputTokens;
|
||||
|
||||
const risks = await runPass(gateway, "detect-risks", userPayload, risksJsonSchema);
|
||||
totalIn += risks.inputTokens;
|
||||
totalOut += risks.outputTokens;
|
||||
|
||||
const incons = await runPass(gateway, "detect-inconsistencies", userPayload, inconsistenciesJsonSchema);
|
||||
totalIn += incons.inputTokens;
|
||||
totalOut += incons.outputTokens;
|
||||
|
||||
const raw: Finding[] = [
|
||||
...assumptions.findings.map((f): Finding => ({
|
||||
kind: "assumption",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.5),
|
||||
})),
|
||||
...risks.findings.map((f): Finding => ({
|
||||
kind: "risk",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.5),
|
||||
severity: f.severity ?? "medium",
|
||||
})),
|
||||
...incons.findings.map((f): Finding => ({
|
||||
kind: "inconsistency",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.6),
|
||||
validationCode: f.validationCode,
|
||||
})),
|
||||
];
|
||||
|
||||
const validated = postValidate(raw, model);
|
||||
|
||||
return {
|
||||
findings: validated.findings,
|
||||
inputTokens: totalIn,
|
||||
outputTokens: totalOut,
|
||||
durationMs: Date.now() - start,
|
||||
strippedRefs: validated.strippedRefs,
|
||||
droppedFindings: validated.droppedFindings,
|
||||
provider: gateway.provider,
|
||||
model: gateway.model,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Per-pass runner ─────────────────────────────────────────────────────
|
||||
|
||||
interface PassResult {
|
||||
findings: RawFinding[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
async function runPass(
|
||||
gateway: ReturnType<typeof defaultGateway>,
|
||||
promptName: "detect-assumptions" | "detect-risks" | "detect-inconsistencies",
|
||||
userPayload: string,
|
||||
jsonSchema: object
|
||||
): Promise<PassResult> {
|
||||
const prompt = loadPrompt(`socrates/${promptName}.md`);
|
||||
const messages: Message[] = [
|
||||
{ role: "system", content: prompt },
|
||||
{ role: "user", content: userPayload },
|
||||
];
|
||||
|
||||
try {
|
||||
const { value, result } = await chatJSON<{ findings?: RawFinding[] }>(gateway, messages, {
|
||||
temperature: 0.2,
|
||||
maxTokens: 768,
|
||||
jsonSchema: { name: "findings", schema: jsonSchema as Record<string, unknown> },
|
||||
jsonObjectMode: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
return {
|
||||
findings: Array.isArray(value?.findings) ? value.findings : [],
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
};
|
||||
} catch (err) {
|
||||
// Fail-soft: a busted detection pass shouldn't take down the whole detect.
|
||||
console.error(`[detect] ${promptName} pass failed:`, (err as Error).message);
|
||||
return { findings: [], inputTokens: 0, outputTokens: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Trimmed payload (small models need tight context) ───────────────────
|
||||
|
||||
function buildUserPayload(model: SysMLModel): string {
|
||||
const compact = {
|
||||
blocks: model.blocks.map(b => ({
|
||||
id: b.id,
|
||||
label: b.label,
|
||||
kind: b.kind,
|
||||
properties: b.properties.map(p => p.name),
|
||||
})),
|
||||
associations: model.associations.map(a => ({
|
||||
id: a.id,
|
||||
from: a.fromBlockId,
|
||||
to: a.toBlockId,
|
||||
label: a.label,
|
||||
kind: a.kind,
|
||||
})),
|
||||
constraints: model.constraints.map(c => ({
|
||||
id: c.id,
|
||||
label: c.label,
|
||||
appliesTo: c.appliesTo,
|
||||
})),
|
||||
requirements: model.requirements.map(r => ({
|
||||
id: r.id,
|
||||
tag: r.tag,
|
||||
text: r.text,
|
||||
satisfiedBy: r.relations
|
||||
.filter(rel => rel.kind === "satisfy")
|
||||
.map(rel => (rel as { kind: "satisfy"; blockId: string }).blockId),
|
||||
})),
|
||||
};
|
||||
return `Model:\n\`\`\`json\n${JSON.stringify(compact, null, 2)}\n\`\`\`\n`;
|
||||
}
|
||||
|
||||
// ─── Post-validation: strip hallucinated element refs ────────────────────
|
||||
|
||||
interface ValidateOut {
|
||||
findings: Finding[];
|
||||
strippedRefs: number;
|
||||
droppedFindings: number;
|
||||
}
|
||||
|
||||
function postValidate(findings: Finding[], model: SysMLModel): ValidateOut {
|
||||
const validIds = collectValidIds(model);
|
||||
let stripped = 0;
|
||||
let dropped = 0;
|
||||
const out: Finding[] = [];
|
||||
|
||||
for (const f of findings) {
|
||||
if (!f.text || f.text.trim().length === 0) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
const goodRefs: string[] = [];
|
||||
for (const ref of f.linkedElementIds) {
|
||||
if (validIds.has(ref)) goodRefs.push(ref);
|
||||
else stripped++;
|
||||
}
|
||||
// Drop only if it had refs and ALL of them were hallucinated.
|
||||
if (f.linkedElementIds.length > 0 && goodRefs.length === 0) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
out.push({ ...f, linkedElementIds: goodRefs });
|
||||
}
|
||||
return { findings: out, strippedRefs: stripped, droppedFindings: dropped };
|
||||
}
|
||||
|
||||
function collectValidIds(model: SysMLModel): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
for (const b of model.blocks) {
|
||||
ids.add(b.id);
|
||||
for (const p of b.properties) {
|
||||
ids.add(`${b.id}.${p.id}`);
|
||||
ids.add(p.name); // tolerate bare property names — common LLM pattern
|
||||
}
|
||||
}
|
||||
for (const a of model.associations) ids.add(a.id);
|
||||
for (const c of model.constraints) ids.add(c.id);
|
||||
for (const r of model.requirements) {
|
||||
ids.add(r.id);
|
||||
ids.add(r.tag);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function clamp01(n: number): number {
|
||||
if (Number.isNaN(n)) return 0;
|
||||
return Math.max(0, Math.min(1, n));
|
||||
}
|
||||
Reference in New Issue
Block a user