MVP M8: background detection — assumptions, risks, inconsistencies
The editor now runs Socrates' three Phase-0-validated detection prompts against the live model, persists the findings, and surfaces them in a new FindingsPanel beside the IssuesPanel. Click any finding to focus its linked element across rail + diagram. Re-detect after model edits to refresh against the new state. apps/web/lib/llm/prompts/socrates - detect-assumptions.md / detect-risks.md / detect-inconsistencies.md promoted verbatim from phase-0 (Phase 0 corpus validated them 10/10). apps/web/lib/llm/detect.ts - Three sequential detection passes (parallel was OOM-prone on 4B local models — Phase 0 lesson). Each pass uses the Phase 0 JSON schema with jsonObjectMode fallback + chatJSON repair-retry. Fail-soft per pass: one busted pass returns [] rather than blowing up the whole detect. - post-validate strips hallucinated element refs (drops findings whose refs ALL fail to resolve; keeps findings with zero refs since some inconsistencies are genuinely about absences). apps/web/prisma/schema.prisma - Finding table: kind / text / linkedElementIds (JSON) / confidence / severity / validationCode / status / modelVersion / provider / model. - ResearchFinding table reserved for the Tavily integration that comes next — schema in place so we don't have to migrate again. apps/web/lib/db/repo.ts - listOpenFindings(projectId), replaceFindings(...) — replaceFindings wipes prior open findings in a transaction and writes the new set so re-detect doesn't accumulate stale findings. apps/web/app/api/projects/[projectId]/findings/route.ts - GET returns persisted open findings. - POST runs detect, persists, returns findings + meta (provider, model, durationMs, strippedRefs, droppedFindings). apps/web/components/editor/FindingsPanel.tsx - New panel, anchored bottom-right just left of IssuesPanel. Shows count summary (asm / risk / inc), detect / re-detect button, list grouped by kind (inconsistencies first, then risks, then assumptions), per-finding glyph + tag + severity + confidence + linked refs. - Click a finding row → focus its first linked element via the same setFocusBlockId path the rail and IssuesPanel already use. - "stale" indicator when the model version has advanced past the one the findings were detected against. EditorShell wires version + projectId through to FindingsPanel. Smoke-tested end-to-end: 12 findings returned (5 asm / 4 risk / 3 inc), 0 hallucinated refs stripped, ~37s on local gemma-4-e4b. Sample assumption "students are willing to engage with an AI tutor that is programmed to refuse providing complete solutions" — specific to Aristotle's refusal_policy, not a generic startup truism. Deferred to follow-ups: inline rail/diagram badges from findings, auto-detect-on-save, Tavily research, experiment modal.
This commit is contained in:
60
apps/web/app/api/projects/[projectId]/findings/route.ts
Normal file
60
apps/web/app/api/projects/[projectId]/findings/route.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
// GET /api/projects/[id]/findings — returns the project's open findings
|
||||
// POST /api/projects/[id]/findings — runs detection, persists, returns the new set
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
import { loadProject, listOpenFindings, replaceFindings } from "../../../../../lib/db/repo";
|
||||
import { detectFindings } from "../../../../../lib/llm/detect";
|
||||
|
||||
export async function GET(_req: Request, { params }: { params: Promise<{ projectId: string }> }) {
|
||||
const { projectId } = await params;
|
||||
const findings = await listOpenFindings(projectId);
|
||||
return NextResponse.json({
|
||||
findings: findings.map(f => ({
|
||||
id: f.id,
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds,
|
||||
confidence: f.confidence,
|
||||
severity: f.severity,
|
||||
validationCode: f.validationCode,
|
||||
modelVersion: f.modelVersion,
|
||||
provider: f.provider,
|
||||
llmModel: f.llmModel,
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
export async function POST(_req: Request, { params }: { params: Promise<{ projectId: string }> }) {
|
||||
const { projectId } = await params;
|
||||
try {
|
||||
const { model, version } = await loadProject(projectId);
|
||||
const result = await detectFindings(model);
|
||||
const stored = await replaceFindings(projectId, result.findings, version, result.provider, result.model);
|
||||
|
||||
return NextResponse.json({
|
||||
findings: stored.map(f => ({
|
||||
id: f.id,
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds,
|
||||
confidence: f.confidence,
|
||||
severity: f.severity,
|
||||
validationCode: f.validationCode,
|
||||
modelVersion: f.modelVersion,
|
||||
provider: f.provider,
|
||||
llmModel: f.llmModel,
|
||||
})),
|
||||
meta: {
|
||||
provider: result.provider,
|
||||
model: result.model,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
durationMs: result.durationMs,
|
||||
strippedRefs: result.strippedRefs,
|
||||
droppedFindings: result.droppedFindings,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
return NextResponse.json({ error: (err as Error).message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { LeftRail } from "./LeftRail";
|
||||
import { CanvasHeader } from "./CanvasHeader";
|
||||
import { StatusBar } from "./StatusBar";
|
||||
import { IssuesPanel } from "./IssuesPanel";
|
||||
import { FindingsPanel } from "./FindingsPanel";
|
||||
import { TextCanvas } from "../text-canvas/TextCanvas";
|
||||
import { DiagramCanvas, type DiagramVariant } from "../diagram-canvas/DiagramCanvas";
|
||||
import { SocratesDock, type Density, type SocratesPresence } from "../socrates/SocratesDock";
|
||||
@@ -98,7 +99,7 @@ interface ShellBodyProps {
|
||||
|
||||
function ShellBody({ data, projectId, density, markupStyle, diagramStyle, presence, breaks }: ShellBodyProps) {
|
||||
const [focusBlockId, setFocusBlockId] = useState<string | null>(null);
|
||||
const { model, issues, issuesByElement } = useModelStore();
|
||||
const { model, version, issues, issuesByElement } = useModelStore();
|
||||
|
||||
const stats = `SysML · ${model.blocks.length} blocks · ${model.associations.length} associations · ${model.constraints.length} constraints`;
|
||||
const subtitle = breaks.length > 0 ? `${stats} · breaks active: ${breaks.join(", ")}` : stats;
|
||||
@@ -161,6 +162,11 @@ function ShellBody({ data, projectId, density, markupStyle, diagramStyle, presen
|
||||
<StatusBar data={data} />
|
||||
|
||||
<IssuesPanel issues={issues} onSelectAnchor={id => setFocusBlockId(id)} />
|
||||
<FindingsPanel
|
||||
projectId={projectId ?? "aristotle"}
|
||||
modelVersion={version}
|
||||
onSelectAnchor={id => setFocusBlockId(id)}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
187
apps/web/components/editor/FindingsPanel.tsx
Normal file
187
apps/web/components/editor/FindingsPanel.tsx
Normal file
@@ -0,0 +1,187 @@
|
||||
// Findings panel — lists detected assumptions / risks / inconsistencies.
|
||||
// Lives in the bottom-right corner above the IssuesPanel; collapsible;
|
||||
// has a "detect" button that triggers a fresh background pass.
|
||||
//
|
||||
// Items are clickable: clicking focuses the first linked element across
|
||||
// the rail + diagram (using the same setFocusBlockId path as the rail).
|
||||
|
||||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
|
||||
export type FindingKind = "assumption" | "risk" | "inconsistency";
|
||||
|
||||
export interface FindingDTO {
|
||||
id: string;
|
||||
kind: FindingKind;
|
||||
text: string;
|
||||
linkedElementIds: string[];
|
||||
confidence: number;
|
||||
severity?: "low" | "medium" | "high" | null;
|
||||
validationCode?: string | null;
|
||||
modelVersion: number;
|
||||
provider?: string | null;
|
||||
llmModel?: string | null;
|
||||
}
|
||||
|
||||
interface FindingsPanelProps {
|
||||
projectId: string;
|
||||
/** Re-fetch whenever this changes. Pass the model version so we know when to refresh. */
|
||||
modelVersion: number;
|
||||
onSelectAnchor?: (elementId: string) => void;
|
||||
}
|
||||
|
||||
const KIND_GLYPH: Record<FindingKind, string> = {
|
||||
assumption: "●",
|
||||
risk: "▲",
|
||||
inconsistency: "!",
|
||||
};
|
||||
|
||||
const KIND_LABEL: Record<FindingKind, string> = {
|
||||
assumption: "asm",
|
||||
risk: "risk",
|
||||
inconsistency: "inc",
|
||||
};
|
||||
|
||||
export function FindingsPanel({ projectId, modelVersion, onSelectAnchor }: FindingsPanelProps) {
|
||||
const [findings, setFindings] = useState<FindingDTO[] | null>(null);
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [running, setRunning] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [meta, setMeta] = useState<{ provider?: string; model?: string; durationMs?: number; modelVersion?: number } | null>(null);
|
||||
|
||||
// Initial load
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
(async () => {
|
||||
try {
|
||||
const res = await fetch(`/api/projects/${encodeURIComponent(projectId)}/findings`);
|
||||
if (!res.ok) throw new Error(String(res.status));
|
||||
const data = (await res.json()) as { findings: FindingDTO[] };
|
||||
if (cancelled) return;
|
||||
setFindings(data.findings);
|
||||
if (data.findings.length > 0) {
|
||||
const first = data.findings[0]!;
|
||||
setMeta({ provider: first.provider ?? undefined, model: first.llmModel ?? undefined, modelVersion: first.modelVersion });
|
||||
}
|
||||
} catch (err) {
|
||||
if (!cancelled) setError((err as Error).message);
|
||||
}
|
||||
})();
|
||||
return () => { cancelled = true; };
|
||||
}, [projectId]);
|
||||
|
||||
const detect = useCallback(async () => {
|
||||
if (running) return;
|
||||
setRunning(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await fetch(`/api/projects/${encodeURIComponent(projectId)}/findings`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (!res.ok) {
|
||||
const body = await res.json().catch(() => ({}));
|
||||
throw new Error(body.error ?? `HTTP ${res.status}`);
|
||||
}
|
||||
const data = (await res.json()) as {
|
||||
findings: FindingDTO[];
|
||||
meta: { provider: string; model: string; durationMs: number; strippedRefs: number; droppedFindings: number };
|
||||
};
|
||||
setFindings(data.findings);
|
||||
setMeta({
|
||||
provider: data.meta.provider,
|
||||
model: data.meta.model,
|
||||
durationMs: data.meta.durationMs,
|
||||
modelVersion,
|
||||
});
|
||||
} catch (err) {
|
||||
setError((err as Error).message);
|
||||
} finally {
|
||||
setRunning(false);
|
||||
}
|
||||
}, [projectId, modelVersion, running]);
|
||||
|
||||
const counts = {
|
||||
assumption: findings?.filter(f => f.kind === "assumption").length ?? 0,
|
||||
risk: findings?.filter(f => f.kind === "risk").length ?? 0,
|
||||
inconsistency: findings?.filter(f => f.kind === "inconsistency").length ?? 0,
|
||||
};
|
||||
const total = counts.assumption + counts.risk + counts.inconsistency;
|
||||
const stale = meta?.modelVersion !== undefined && meta.modelVersion !== modelVersion;
|
||||
|
||||
return (
|
||||
<div className={`findings-panel ${collapsed ? "findings-panel-collapsed" : ""}`}>
|
||||
<button
|
||||
type="button"
|
||||
className="findings-panel-head"
|
||||
onClick={() => setCollapsed(c => !c)}
|
||||
>
|
||||
<span className="findings-panel-counts">
|
||||
{counts.assumption > 0 && <span className="findings-count findings-count-asm">● {counts.assumption}</span>}
|
||||
{counts.risk > 0 && <span className="findings-count findings-count-risk">▲ {counts.risk}</span>}
|
||||
{counts.inconsistency > 0 && <span className="findings-count findings-count-inc">! {counts.inconsistency}</span>}
|
||||
{total === 0 && findings !== null && <span className="findings-count findings-count-none">no findings</span>}
|
||||
{findings === null && <span className="findings-count findings-count-none">loading…</span>}
|
||||
</span>
|
||||
<span className="findings-panel-title">
|
||||
{findings === null ? "findings" : total === 0 ? "no detected findings" : `${total} detected finding${total === 1 ? "" : "s"}`}
|
||||
{stale && <span className="findings-panel-stale" title="Model has changed since these were detected">· stale</span>}
|
||||
</span>
|
||||
<span className="findings-panel-caret">{collapsed ? "▴" : "▾"}</span>
|
||||
</button>
|
||||
|
||||
{!collapsed && (
|
||||
<>
|
||||
<div className="findings-panel-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="findings-panel-btn"
|
||||
onClick={detect}
|
||||
disabled={running}
|
||||
>
|
||||
{running ? "detecting…" : findings && findings.length > 0 ? "re-detect" : "detect"}
|
||||
</button>
|
||||
{meta?.durationMs && (
|
||||
<span className="findings-panel-meta">
|
||||
{meta.provider} · {meta.model?.split("/").pop()} · {(meta.durationMs / 1000).toFixed(1)}s
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{error && <div className="findings-panel-error">⚠ {error}</div>}
|
||||
|
||||
{findings && findings.length > 0 && (
|
||||
<ul className="findings-panel-list">
|
||||
{(["inconsistency", "risk", "assumption"] as const).flatMap(kind =>
|
||||
findings.filter(f => f.kind === kind).map(f => (
|
||||
<li
|
||||
key={f.id}
|
||||
className={`findings-item findings-item-${f.kind}`}
|
||||
onClick={() => {
|
||||
if (f.linkedElementIds.length > 0 && onSelectAnchor) {
|
||||
onSelectAnchor(f.linkedElementIds[0]!);
|
||||
}
|
||||
}}
|
||||
style={{ cursor: f.linkedElementIds.length > 0 ? "pointer" : "default" }}
|
||||
>
|
||||
<span className={`findings-item-glyph findings-item-glyph-${f.kind}`}>{KIND_GLYPH[f.kind]}</span>
|
||||
<span className="findings-item-tag">{KIND_LABEL[f.kind]}</span>
|
||||
{f.severity && <span className={`findings-item-sev findings-item-sev-${f.severity}`}>{f.severity}</span>}
|
||||
{f.validationCode && <span className="findings-item-code">{f.validationCode}</span>}
|
||||
<span className="findings-item-text">{f.text}</span>
|
||||
<span className="findings-item-conf">{f.confidence.toFixed(2)}</span>
|
||||
{f.linkedElementIds.length > 0 && (
|
||||
<span className="findings-item-refs" title={f.linkedElementIds.join(", ")}>
|
||||
[{f.linkedElementIds.slice(0, 3).join(", ")}{f.linkedElementIds.length > 3 ? "…" : ""}]
|
||||
</span>
|
||||
)}
|
||||
</li>
|
||||
))
|
||||
)}
|
||||
</ul>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -148,6 +148,114 @@ async function ensureSeeded(projectId: string): Promise<{ projectId: string }> {
|
||||
return { projectId };
|
||||
}
|
||||
|
||||
// ─── Findings (M8) ───────────────────────────────────────────────────────
|
||||
|
||||
import type { Finding as DetectedFinding } from "../llm/detect";
|
||||
|
||||
export interface StoredFinding {
|
||||
id: string;
|
||||
kind: string;
|
||||
text: string;
|
||||
linkedElementIds: string[];
|
||||
confidence: number;
|
||||
severity: string | null;
|
||||
validationCode: string | null;
|
||||
status: string;
|
||||
modelVersion: number;
|
||||
provider: string | null;
|
||||
llmModel: string | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export async function listOpenFindings(projectId: string): Promise<StoredFinding[]> {
|
||||
const rows = await prisma.finding.findMany({
|
||||
where: { projectId, status: "open" },
|
||||
orderBy: [{ kind: "asc" }, { createdAt: "desc" }],
|
||||
});
|
||||
return rows.map(rowToFinding);
|
||||
}
|
||||
|
||||
export async function replaceFindings(
|
||||
projectId: string,
|
||||
findings: DetectedFinding[],
|
||||
modelVersion: number,
|
||||
provider: string,
|
||||
llmModel: string
|
||||
): Promise<StoredFinding[]> {
|
||||
return prisma.$transaction(async tx => {
|
||||
// Wipe previous open findings — re-running detection supersedes them.
|
||||
await tx.finding.deleteMany({ where: { projectId, status: "open" } });
|
||||
if (findings.length === 0) return [];
|
||||
|
||||
const created = await Promise.all(
|
||||
findings.map(f =>
|
||||
tx.finding.create({
|
||||
data: {
|
||||
projectId,
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
linkedElementIds: JSON.stringify(f.linkedElementIds),
|
||||
confidence: f.confidence,
|
||||
severity: f.severity ?? null,
|
||||
validationCode: f.validationCode ?? null,
|
||||
status: "open",
|
||||
modelVersion,
|
||||
provider,
|
||||
llmModel,
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
return created.map(rowToFinding);
|
||||
});
|
||||
}
|
||||
|
||||
export async function dismissFinding(findingId: string): Promise<void> {
|
||||
await prisma.finding.update({
|
||||
where: { id: findingId },
|
||||
data: { status: "dismissed" },
|
||||
});
|
||||
}
|
||||
|
||||
interface FindingRow {
|
||||
id: string;
|
||||
kind: string;
|
||||
text: string;
|
||||
linkedElementIds: string;
|
||||
confidence: number;
|
||||
severity: string | null;
|
||||
validationCode: string | null;
|
||||
status: string;
|
||||
modelVersion: number;
|
||||
provider: string | null;
|
||||
llmModel: string | null;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
function rowToFinding(row: FindingRow): StoredFinding {
|
||||
let linked: string[] = [];
|
||||
try {
|
||||
const parsed = JSON.parse(row.linkedElementIds);
|
||||
if (Array.isArray(parsed)) linked = parsed.filter((x): x is string => typeof x === "string");
|
||||
} catch {
|
||||
linked = [];
|
||||
}
|
||||
return {
|
||||
id: row.id,
|
||||
kind: row.kind,
|
||||
text: row.text,
|
||||
linkedElementIds: linked,
|
||||
confidence: row.confidence,
|
||||
severity: row.severity,
|
||||
validationCode: row.validationCode,
|
||||
status: row.status,
|
||||
modelVersion: row.modelVersion,
|
||||
provider: row.provider,
|
||||
llmModel: row.llmModel,
|
||||
createdAt: row.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Socrates threads ────────────────────────────────────────────────────
|
||||
|
||||
export async function getActiveThread(projectId: string): Promise<{ id: string; title: string | null; messages: Array<{ id: string; role: string; content: string; createdAt: Date }> } | null> {
|
||||
|
||||
301
apps/web/lib/llm/detect.ts
Normal file
301
apps/web/lib/llm/detect.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
// Background detection — runs three Phase-0-validated detection prompts
|
||||
// against the current model and returns structured findings.
|
||||
//
|
||||
// Sequential not parallel: small local models share a KV-cache budget; three
|
||||
// concurrent calls can OOM. Each call is independent so total wall time
|
||||
// roughly equals 3× per-call time, but stability is much higher.
|
||||
//
|
||||
// Post-validation strips hallucinated element ids — the same defensive step
|
||||
// that rescued ~5% of phase-0 detection runs.
|
||||
|
||||
import "server-only";
|
||||
import { defaultGateway, chatJSON, type Message } from "./gateway";
|
||||
import { loadPrompt } from "./prompts";
|
||||
import type { SysMLModel } from "../sysml/model";
|
||||
|
||||
// ─── Output types ────────────────────────────────────────────────────────
|
||||
|
||||
export type FindingKind = "assumption" | "risk" | "inconsistency";
|
||||
|
||||
export interface Finding {
|
||||
kind: FindingKind;
|
||||
text: string;
|
||||
linkedElementIds: string[];
|
||||
confidence: number;
|
||||
severity?: "low" | "medium" | "high";
|
||||
validationCode?: string;
|
||||
}
|
||||
|
||||
export interface DetectResult {
|
||||
findings: Finding[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
strippedRefs: number;
|
||||
droppedFindings: number;
|
||||
provider: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
// ─── Per-pass JSON schemas (oneOf-by-shape; small models cope) ──────────
|
||||
|
||||
const baseProps = {
|
||||
text: { type: "string", minLength: 1 },
|
||||
linkedElementIds: { type: "array", items: { type: "string" } },
|
||||
confidence: { type: "number", minimum: 0, maximum: 1 },
|
||||
};
|
||||
|
||||
const assumptionsJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence"],
|
||||
properties: baseProps,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const risksJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence", "severity"],
|
||||
properties: {
|
||||
...baseProps,
|
||||
severity: { type: "string", enum: ["low", "medium", "high"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const inconsistenciesJsonSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["findings"],
|
||||
properties: {
|
||||
findings: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["text", "confidence"],
|
||||
properties: {
|
||||
...baseProps,
|
||||
validationCode: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
interface RawFinding {
|
||||
text?: string;
|
||||
linkedElementIds?: string[];
|
||||
confidence?: number;
|
||||
severity?: "low" | "medium" | "high";
|
||||
validationCode?: string;
|
||||
}
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export async function detectFindings(model: SysMLModel): Promise<DetectResult> {
|
||||
const gateway = defaultGateway();
|
||||
const start = Date.now();
|
||||
let totalIn = 0;
|
||||
let totalOut = 0;
|
||||
|
||||
const userPayload = buildUserPayload(model);
|
||||
|
||||
const assumptions = await runPass(gateway, "detect-assumptions", userPayload, assumptionsJsonSchema);
|
||||
totalIn += assumptions.inputTokens;
|
||||
totalOut += assumptions.outputTokens;
|
||||
|
||||
const risks = await runPass(gateway, "detect-risks", userPayload, risksJsonSchema);
|
||||
totalIn += risks.inputTokens;
|
||||
totalOut += risks.outputTokens;
|
||||
|
||||
const incons = await runPass(gateway, "detect-inconsistencies", userPayload, inconsistenciesJsonSchema);
|
||||
totalIn += incons.inputTokens;
|
||||
totalOut += incons.outputTokens;
|
||||
|
||||
const raw: Finding[] = [
|
||||
...assumptions.findings.map((f): Finding => ({
|
||||
kind: "assumption",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.5),
|
||||
})),
|
||||
...risks.findings.map((f): Finding => ({
|
||||
kind: "risk",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.5),
|
||||
severity: f.severity ?? "medium",
|
||||
})),
|
||||
...incons.findings.map((f): Finding => ({
|
||||
kind: "inconsistency",
|
||||
text: f.text ?? "",
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: clamp01(f.confidence ?? 0.6),
|
||||
validationCode: f.validationCode,
|
||||
})),
|
||||
];
|
||||
|
||||
const validated = postValidate(raw, model);
|
||||
|
||||
return {
|
||||
findings: validated.findings,
|
||||
inputTokens: totalIn,
|
||||
outputTokens: totalOut,
|
||||
durationMs: Date.now() - start,
|
||||
strippedRefs: validated.strippedRefs,
|
||||
droppedFindings: validated.droppedFindings,
|
||||
provider: gateway.provider,
|
||||
model: gateway.model,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Per-pass runner ─────────────────────────────────────────────────────
|
||||
|
||||
interface PassResult {
|
||||
findings: RawFinding[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
async function runPass(
|
||||
gateway: ReturnType<typeof defaultGateway>,
|
||||
promptName: "detect-assumptions" | "detect-risks" | "detect-inconsistencies",
|
||||
userPayload: string,
|
||||
jsonSchema: object
|
||||
): Promise<PassResult> {
|
||||
const prompt = loadPrompt(`socrates/${promptName}.md`);
|
||||
const messages: Message[] = [
|
||||
{ role: "system", content: prompt },
|
||||
{ role: "user", content: userPayload },
|
||||
];
|
||||
|
||||
try {
|
||||
const { value, result } = await chatJSON<{ findings?: RawFinding[] }>(gateway, messages, {
|
||||
temperature: 0.2,
|
||||
maxTokens: 768,
|
||||
jsonSchema: { name: "findings", schema: jsonSchema as Record<string, unknown> },
|
||||
jsonObjectMode: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
return {
|
||||
findings: Array.isArray(value?.findings) ? value.findings : [],
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
};
|
||||
} catch (err) {
|
||||
// Fail-soft: a busted detection pass shouldn't take down the whole detect.
|
||||
console.error(`[detect] ${promptName} pass failed:`, (err as Error).message);
|
||||
return { findings: [], inputTokens: 0, outputTokens: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Trimmed payload (small models need tight context) ───────────────────
|
||||
|
||||
function buildUserPayload(model: SysMLModel): string {
|
||||
const compact = {
|
||||
blocks: model.blocks.map(b => ({
|
||||
id: b.id,
|
||||
label: b.label,
|
||||
kind: b.kind,
|
||||
properties: b.properties.map(p => p.name),
|
||||
})),
|
||||
associations: model.associations.map(a => ({
|
||||
id: a.id,
|
||||
from: a.fromBlockId,
|
||||
to: a.toBlockId,
|
||||
label: a.label,
|
||||
kind: a.kind,
|
||||
})),
|
||||
constraints: model.constraints.map(c => ({
|
||||
id: c.id,
|
||||
label: c.label,
|
||||
appliesTo: c.appliesTo,
|
||||
})),
|
||||
requirements: model.requirements.map(r => ({
|
||||
id: r.id,
|
||||
tag: r.tag,
|
||||
text: r.text,
|
||||
satisfiedBy: r.relations
|
||||
.filter(rel => rel.kind === "satisfy")
|
||||
.map(rel => (rel as { kind: "satisfy"; blockId: string }).blockId),
|
||||
})),
|
||||
};
|
||||
return `Model:\n\`\`\`json\n${JSON.stringify(compact, null, 2)}\n\`\`\`\n`;
|
||||
}
|
||||
|
||||
// ─── Post-validation: strip hallucinated element refs ────────────────────
|
||||
|
||||
interface ValidateOut {
|
||||
findings: Finding[];
|
||||
strippedRefs: number;
|
||||
droppedFindings: number;
|
||||
}
|
||||
|
||||
function postValidate(findings: Finding[], model: SysMLModel): ValidateOut {
|
||||
const validIds = collectValidIds(model);
|
||||
let stripped = 0;
|
||||
let dropped = 0;
|
||||
const out: Finding[] = [];
|
||||
|
||||
for (const f of findings) {
|
||||
if (!f.text || f.text.trim().length === 0) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
const goodRefs: string[] = [];
|
||||
for (const ref of f.linkedElementIds) {
|
||||
if (validIds.has(ref)) goodRefs.push(ref);
|
||||
else stripped++;
|
||||
}
|
||||
// Drop only if it had refs and ALL of them were hallucinated.
|
||||
if (f.linkedElementIds.length > 0 && goodRefs.length === 0) {
|
||||
dropped++;
|
||||
continue;
|
||||
}
|
||||
out.push({ ...f, linkedElementIds: goodRefs });
|
||||
}
|
||||
return { findings: out, strippedRefs: stripped, droppedFindings: dropped };
|
||||
}
|
||||
|
||||
function collectValidIds(model: SysMLModel): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
for (const b of model.blocks) {
|
||||
ids.add(b.id);
|
||||
for (const p of b.properties) {
|
||||
ids.add(`${b.id}.${p.id}`);
|
||||
ids.add(p.name); // tolerate bare property names — common LLM pattern
|
||||
}
|
||||
}
|
||||
for (const a of model.associations) ids.add(a.id);
|
||||
for (const c of model.constraints) ids.add(c.id);
|
||||
for (const r of model.requirements) {
|
||||
ids.add(r.id);
|
||||
ids.add(r.tag);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function clamp01(n: number): number {
|
||||
if (Number.isNaN(n)) return 0;
|
||||
return Math.max(0, Math.min(1, n));
|
||||
}
|
||||
50
apps/web/lib/llm/prompts/socrates/detect-assumptions.md
Normal file
50
apps/web/lib/llm/prompts/socrates/detect-assumptions.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Detect implicit assumptions in a product seed and model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: surface the **implicit assumptions** the user is making — beliefs treated as true without explicit validation.
|
||||
|
||||
## What is an assumption
|
||||
|
||||
A measurable, falsifiable belief that underpins the idea but isn't stated as a requirement or constraint. Examples:
|
||||
|
||||
- "Students will accept a tool that refuses to answer" — assumes adoption willingness
|
||||
- "1.2s P50 latency is achievable on-prem with available models" — assumes technical feasibility
|
||||
- "Faculty will not classify Socratic prompts as academic dishonesty" — assumes institutional acceptance
|
||||
|
||||
## What is NOT an assumption
|
||||
|
||||
- Stated requirements (REQ-NNN entries) — those are explicit goals
|
||||
- Constraints — those are non-negotiables, not beliefs
|
||||
- Definitions of terms
|
||||
- Generic startup truisms ("users will want this") — too vague to be a useful assumption
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings` — an array of assumption candidates. Each candidate:
|
||||
|
||||
- `text` — the assumption restated cleanly, in one sentence, in the user's register
|
||||
- `linkedElementIds` — array of model element ids this assumption is about (block ids, requirement ids, or constraint ids — must match what's in the model)
|
||||
- `confidence` — 0.0 to 1.0, how confident you are this is genuinely an unstated assumption
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.5`
|
||||
- Cap at 8 findings
|
||||
- Each assumption must name a SPECIFIC, falsifiable belief — not a generic concern
|
||||
- Each must reference at least one real element id from the model
|
||||
- If the seed is sparse and you cannot surface real assumptions, return fewer (or none) rather than fabricating
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
45
apps/web/lib/llm/prompts/socrates/detect-inconsistencies.md
Normal file
45
apps/web/lib/llm/prompts/socrates/detect-inconsistencies.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# Detect inconsistencies in a generated model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: find **inconsistencies** — internal contradictions or structural problems in the model.
|
||||
|
||||
## Categories
|
||||
|
||||
- **Internal contradictions** — two requirements that can't both hold simultaneously; a block whose properties contradict its kind; a constraint already violated by some property value
|
||||
- **Reference issues** — an association whose endpoints don't make semantic sense (e.g., actor → constraint, or system → external actor with the wrong direction)
|
||||
- **Over-broad claims** — a requirement that promises more than the system can deliver based on the blocks present
|
||||
- **Missing satisfiers** — a requirement with no plausible block to satisfy it
|
||||
- **Unused elements** — a block with no associations and no requirement satisfaction (may be dead)
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings`. Each candidate:
|
||||
|
||||
- `text` — the inconsistency stated clearly in one sentence
|
||||
- `linkedElementIds` — array of element ids involved
|
||||
- `confidence` — 0.0 to 1.0
|
||||
- `validationCode` — optional. If the issue matches a structural rule, include the code: `M2` (cyclic composition), `T1` (untraced requirement), `T2` (unused element), `S1` (dangling association endpoint). Otherwise omit.
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.6` — for inconsistencies, false positives are worse than misses
|
||||
- Cap at 6 findings
|
||||
- An inconsistency must point to a SPECIFIC contradiction or structural defect, not a stylistic preference
|
||||
- "This block has too many properties" is NOT an inconsistency
|
||||
- "Requirement REQ-002 forbids what association A2 enables" IS an inconsistency
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"confidence": 0.0,
|
||||
"validationCode": "string (optional)"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
46
apps/web/lib/llm/prompts/socrates/detect-risks.md
Normal file
46
apps/web/lib/llm/prompts/socrates/detect-risks.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# Detect risks in a product seed and model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: surface **risks** — specific failure modes that could prevent the system from working as intended.
|
||||
|
||||
## Risk categories
|
||||
|
||||
- **Technical** — feasibility, performance, scaling
|
||||
- **Market** — adoption, competitive dynamics, distribution
|
||||
- **Execution** — team capability, timing, dependencies
|
||||
- **Regulatory** — compliance, legal, privacy
|
||||
- **External** — third-party reliance, geopolitical, supply
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings` — an array of risk candidates. Each candidate:
|
||||
|
||||
- `text` — the risk restated as a specific failure mode in one sentence
|
||||
- `linkedElementIds` — array of model element ids this risk implicates
|
||||
- `severity` — `"low"`, `"medium"`, or `"high"` (impact-if-it-happens, NOT probability)
|
||||
- `confidence` — 0.0 to 1.0, how confident you are this is a real risk worth tracking
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.5`
|
||||
- Cap at 6 findings
|
||||
- A risk must name a SPECIFIC failure mode tied to SPECIFIC element(s). "Won't work" is not a risk; "Latency target unachievable on consumer-grade hardware given 7B-param inference" is.
|
||||
- Severity reflects what happens IF the risk materializes, not how likely it is.
|
||||
- Each finding must reference at least one real element id from the model
|
||||
- For vague seeds with weak models, return fewer findings rather than fabricated ones
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"severity": "low" | "medium" | "high",
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
@@ -26,6 +26,7 @@ model Project {
|
||||
snapshots ModelSnapshot[]
|
||||
changes ChangelogEntry[]
|
||||
threads SocratesThread[]
|
||||
findings Finding[]
|
||||
}
|
||||
|
||||
model ModelSnapshot {
|
||||
@@ -87,3 +88,55 @@ model SocratesMessage {
|
||||
|
||||
@@index([threadId, createdAt])
|
||||
}
|
||||
|
||||
/// Background-detected finding (assumption / risk / inconsistency).
|
||||
/// Findings are scoped to the model version that produced them; on every
|
||||
/// detection run we delete the project's previous open findings and write
|
||||
/// the new set so we don't accumulate stale ones across model edits.
|
||||
model Finding {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
/// "assumption" | "risk" | "inconsistency"
|
||||
kind String
|
||||
text String
|
||||
/// JSON-encoded string[] of element ids this finding references.
|
||||
linkedElementIds String
|
||||
confidence Float
|
||||
/// Risks only: "low" | "medium" | "high"
|
||||
severity String?
|
||||
/// Inconsistencies only: optional structural-rule code (S1, M2, T1, …)
|
||||
validationCode String?
|
||||
/// Lifecycle: "open" | "dismissed" | "resolved"
|
||||
status String @default("open")
|
||||
/// Model version this finding was detected against.
|
||||
modelVersion Int
|
||||
/// Provider + model that emitted the finding.
|
||||
provider String?
|
||||
llmModel String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
research ResearchFinding[]
|
||||
|
||||
@@index([projectId, status])
|
||||
@@index([projectId, kind])
|
||||
}
|
||||
|
||||
/// Web research results from validating a finding via Tavily / similar.
|
||||
/// Deferred to a follow-up commit; schema is here so we don't have to
|
||||
/// migrate again later.
|
||||
model ResearchFinding {
|
||||
id String @id @default(cuid())
|
||||
findingId String
|
||||
query String
|
||||
url String
|
||||
title String?
|
||||
snippet String?
|
||||
/// "supports" | "contradicts" | "neutral"
|
||||
stance String @default("neutral")
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
finding Finding @relation(fields: [findingId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@index([findingId])
|
||||
}
|
||||
|
||||
@@ -1245,3 +1245,156 @@ button { font-family: inherit; }
|
||||
opacity: 0.4;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* ─── Findings panel (M8) ─── */
|
||||
.findings-panel {
|
||||
position: fixed;
|
||||
/* Sit just above the IssuesPanel; both anchored bottom-right */
|
||||
bottom: 32px;
|
||||
right: 412px;
|
||||
width: 460px;
|
||||
max-height: 50vh;
|
||||
background: var(--surface);
|
||||
border: 1px solid var(--border-strong);
|
||||
border-radius: 6px;
|
||||
box-shadow: 0 8px 24px var(--shadow-strong);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
z-index: 50;
|
||||
font-family: var(--font-body);
|
||||
overflow: hidden;
|
||||
}
|
||||
.findings-panel-collapsed { max-height: none; }
|
||||
.findings-panel-head {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 8px 12px;
|
||||
background: transparent;
|
||||
border: none;
|
||||
border-bottom: 1px solid var(--border);
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
color: var(--fg);
|
||||
width: 100%;
|
||||
text-align: left;
|
||||
}
|
||||
.findings-panel-collapsed .findings-panel-head { border-bottom: none; }
|
||||
.findings-panel-head:hover { background: var(--surface-2); }
|
||||
.findings-panel-counts { display: flex; gap: 8px; }
|
||||
.findings-count {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 11px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.findings-count-asm { color: var(--accent-strong); }
|
||||
.findings-count-risk { color: var(--warn-strong); }
|
||||
.findings-count-inc { color: var(--ok-strong); }
|
||||
.findings-count-none { color: var(--muted); font-style: italic; }
|
||||
.findings-panel-title {
|
||||
font-family: var(--font-display);
|
||||
font-weight: 500;
|
||||
font-size: 13px;
|
||||
flex: 1;
|
||||
}
|
||||
.findings-panel-stale {
|
||||
margin-left: 6px;
|
||||
font-family: var(--font-mono);
|
||||
font-size: 10px;
|
||||
color: var(--warn-strong);
|
||||
}
|
||||
.findings-panel-caret { font-size: 9px; color: var(--muted); }
|
||||
|
||||
.findings-panel-actions {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 6px 12px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
background: var(--surface-2);
|
||||
}
|
||||
.findings-panel-btn {
|
||||
padding: 3px 10px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid var(--accent);
|
||||
background: var(--accent);
|
||||
color: var(--accent-on);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 10.5px;
|
||||
cursor: pointer;
|
||||
}
|
||||
.findings-panel-btn:hover:not(:disabled) { background: var(--accent-strong); border-color: var(--accent-strong); }
|
||||
.findings-panel-btn:disabled { opacity: 0.45; cursor: wait; }
|
||||
.findings-panel-meta {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 10px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.findings-panel-error {
|
||||
padding: 8px 12px;
|
||||
background: var(--warn-soft);
|
||||
color: var(--warn-strong);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 11px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.findings-panel-list {
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 4px;
|
||||
overflow-y: auto;
|
||||
flex: 1;
|
||||
}
|
||||
.findings-item {
|
||||
display: grid;
|
||||
grid-template-columns: 14px 28px auto auto 1fr auto auto;
|
||||
gap: 6px;
|
||||
align-items: baseline;
|
||||
padding: 5px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.findings-item:hover { background: var(--surface-2); }
|
||||
.findings-item-glyph {
|
||||
font-size: 11px;
|
||||
text-align: center;
|
||||
}
|
||||
.findings-item-glyph-assumption { color: var(--accent-strong); }
|
||||
.findings-item-glyph-risk { color: var(--warn-strong); }
|
||||
.findings-item-glyph-inconsistency { color: var(--ok-strong); }
|
||||
.findings-item-tag {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 9.5px;
|
||||
letter-spacing: 0.06em;
|
||||
text-transform: uppercase;
|
||||
color: var(--muted-strong);
|
||||
background: var(--surface-2);
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
.findings-item-sev {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 9.5px;
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
.findings-item-sev-low { background: var(--surface-2); color: var(--muted-strong); }
|
||||
.findings-item-sev-medium { background: var(--accent-soft); color: var(--accent-strong); }
|
||||
.findings-item-sev-high { background: var(--warn-soft); color: var(--warn-strong); }
|
||||
.findings-item-code {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 9.5px;
|
||||
color: var(--muted-strong);
|
||||
background: var(--surface-2);
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
.findings-item-text { color: var(--prose); }
|
||||
.findings-item-conf {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 9.5px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.findings-item-refs {
|
||||
font-family: var(--font-mono);
|
||||
font-size: 9.5px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user