Socrates/phase-0/src/cli.ts

#!/usr/bin/env node
// Phase 0 CLI entrypoint.

import { Command } from 'commander';
import kleur from 'kleur';
import { describeClient, modelName } from './llm/client.js';
import { chat } from './llm/messages.js';
import { toPlantUML } from './render/to-plantuml.js';
import { renderPNG, writePlantUMLSource, plantUMLViewerURL } from './render/render.js';
import { aristotleReferenceModel } from './fixtures/aristotle.js';
import { loadSeed } from './seed/load.js';
import { generateModel } from './generate/generate.js';
import { detectFindings } from './socrates/detect.js';
import { runConversation } from './socrates/converse.js';
import { proposeChange, applyProposal, summarizeOp } from './socrates/propose.js';
import { runCorpus } from './eval/corpus.js';
import { judgeSeed, averageScore, minDimension } from './eval/judge.js';
import * as readline from 'node:readline/promises';
import { stdin as stdInput, stdout as stdOutput } from 'node:process';
import { resolve } from 'node:path';
import { writeFile, mkdir } from 'node:fs/promises';

const program = new Command();
program
  .name('phase0')
  .description('Socrata Phase 0 validation harness')
  .version('0.0.1');

program
  .command('hello')
  .description('Smoke test: round-trip a chat to LM Studio')
  .action(async () => {
    console.log(kleur.dim(`> ${describeClient()}`));
    const start = Date.now();
    try {
      const result = await chat([
        { role: 'system', content: 'You are a terse assistant. Reply in one sentence.' },
        { role: 'user', content: 'Are you there? Identify yourself briefly.' },
      ], { temperature: 0.2, maxTokens: 256 });
      const elapsed = Date.now() - start;
      console.log(kleur.green('✓ round-trip OK'));
      console.log(kleur.dim(`  ${elapsed}ms · ${result.inputTokens} in / ${result.outputTokens} out`));
      console.log();
      console.log(kleur.bold('response:'));
      console.log(result.text);
    } catch (err) {
      console.error(kleur.red('✗ round-trip FAILED'));
      console.error(kleur.red(`  ${(err as Error).message}`));
      console.error(kleur.dim('\nIs LM Studio running? Is a model loaded?'));
      process.exit(1);
    }
  });

program
  .command('render-test')
  .description('Render the hand-written Aristotle reference model to PlantUML PNG')
  .option('-o, --out <dir>', 'output directory', './outputs/_render-test')
  .action(async (opts: { out: string }) => {
    const outDir = resolve(opts.out);
    const pumlPath = `${outDir}/aristotle.puml`;
    const pngPath = `${outDir}/aristotle.png`;

    const source = toPlantUML(aristotleReferenceModel);
    await writePlantUMLSource(source, pumlPath);
    console.log(kleur.green('✓ wrote PlantUML source'));
    console.log(kleur.dim(`  ${pumlPath}`));

    try {
      await renderPNG(source, pngPath);
      console.log(kleur.green('✓ rendered PNG'));
      console.log(kleur.dim(`  ${pngPath}`));
      console.log();
      console.log(kleur.bold('view online:'));
      console.log(plantUMLViewerURL(source));
    } catch (err) {
      console.error(kleur.yellow('⚠ PNG render failed (PlantUML source still saved):'));
      console.error(kleur.dim(`  ${(err as Error).message}`));
      console.log();
      console.log(kleur.bold('view online:'));
      console.log(plantUMLViewerURL(source));
    }
  });

program
  .command('run <seedId>')
  .description('Run a seed through generation + render')
  .option('-o, --out <dir>', 'output directory (default: outputs/<seedId>)')
  .option('--converse', 'after detection, open an interactive Socrates conversation')
  .option('--propose', 'after detection, ask Socrates to propose a model change (accept/reject)')
  .action(async (seedId: string, opts: { out?: string; converse?: boolean; propose?: boolean }) => {
    const outDir = resolve(opts.out ?? `./outputs/${seedId}`);
    console.log(kleur.dim(`> seed: ${seedId}`));
    console.log(kleur.dim(`> ${describeClient()}`));
    console.log();

    let seed;
    try {
      seed = await loadSeed(seedId);
    } catch (err) {
      console.error(kleur.red(`✗ couldn't load seed: ${(err as Error).message}`));
      process.exit(1);
    }
    console.log(kleur.bold(`seed: ${seed.title}`));
    console.log(kleur.dim(`  problem: ${truncate(seed.problem, 120)}`));
    console.log();

    // Generate
    console.log(kleur.cyan('generating model…'));
    const gen = await generateModel(seed);
    console.log(kleur.green('✓ model generated'));
    console.log(
      kleur.dim(
        `  ${gen.durationMs}ms · ${gen.inputTokens} in / ${gen.outputTokens} out · ` +
          `${gen.model.blocks.length} blocks · ${gen.model.associations.length} associations · ` +
          `${gen.model.constraints.length} constraints · ${gen.model.requirements.length} requirements · ` +
          `confidence ${gen.model.overallConfidence.toFixed(2)}`
      )
    );
    if (gen.warnings.length > 0) {
      console.log(kleur.yellow('  lint warnings:'));
      for (const w of gen.warnings) console.log(kleur.yellow(`    · ${w}`));
    }
    console.log();

    // Persist model + render in parallel with detection
    await mkdir(outDir, { recursive: true });
    await writeFile(`${outDir}/model.json`, JSON.stringify(gen.model, null, 2));

    const source = toPlantUML(gen.model);
    await writePlantUMLSource(source, `${outDir}/diagram.puml`);

    // Detection (3 prompts in parallel)
    console.log(kleur.cyan('detecting findings…'));
    const det = await detectFindings(gen.model, seed);
    const counts = countFindings(det.findings);
    console.log(kleur.green('✓ findings detected'));
    console.log(
      kleur.dim(
        `  ${det.durationMs}ms · ${det.inputTokens} in / ${det.outputTokens} out · ` +
          `${counts.assumption} assumptions · ${counts.risk} risks · ${counts.inconsistency} inconsistencies`
      )
    );
    if (det.strippedRefs > 0 || det.droppedFindings > 0) {
      console.log(
        kleur.yellow(
          `  · post-validate: ${det.strippedRefs} hallucinated refs stripped, ${det.droppedFindings} findings dropped`
        )
      );
    }
    await writeFile(`${outDir}/findings.json`, JSON.stringify(det.findings, null, 2));
    if (det.findings.length > 0) {
      console.log();
      printTopFindings(det.findings);
    }
    console.log();

    try {
      await renderPNG(source, `${outDir}/diagram.png`);
      console.log(kleur.green('✓ rendered'));
    } catch (err) {
      console.log(kleur.yellow(`⚠ PNG render failed: ${(err as Error).message}`));
    }
    console.log(kleur.dim(`  ${outDir}/`));
    console.log();
    console.log(kleur.bold('view diagram online:'));
    console.log(plantUMLViewerURL(source));

    // Optional: Socrates proposes a model change (accept/reject)
    if (opts.propose) {
      console.log();
      console.log(kleur.dim('─'.repeat(60)));
      console.log(kleur.bold('Proposal'));
      console.log(kleur.dim('─'.repeat(60)));
      const proposal = await proposeChange(seed, gen.model, det.findings);
      console.log(kleur.bold('Σ Socrates proposes:'));
      console.log(`  ${proposal.reasoning}`);
      console.log();
      if (proposal.ops.length === 0) {
        console.log(kleur.dim('  (no ops — nothing to apply)'));
      } else {
        console.log(kleur.bold('Ops:'));
        for (const op of proposal.ops) console.log(`  ${summarizeOp(op)}`);
        console.log();
        const rl = readline.createInterface({ input: stdInput, output: stdOutput });
        let answer = '';
        try {
          answer = ((await rl.question(kleur.cyan('apply? [y/N] '))) ?? '').trim().toLowerCase();
        } finally {
          rl.close();
        }
        if (answer === 'y' || answer === 'yes') {
          const applied = applyProposal(gen.model, proposal.ops);
          console.log(kleur.green(`✓ applied ${applied.applied.length} op(s)`));
          if (applied.skipped.length > 0) {
            console.log(kleur.yellow(`  ⚠ skipped ${applied.skipped.length}:`));
            for (const s of applied.skipped) console.log(kleur.yellow(`    - ${summarizeOp(s.op)} → ${s.reason}`));
          }
          await writeFile(`${outDir}/model.json`, JSON.stringify(applied.model, null, 2));
          const newSource = toPlantUML(applied.model);
          await writePlantUMLSource(newSource, `${outDir}/diagram.puml`);
          try {
            await renderPNG(newSource, `${outDir}/diagram.png`);
            console.log(kleur.green('✓ re-rendered'));
            console.log(kleur.dim(`  ${outDir}/diagram.png`));
            console.log(kleur.bold('view diagram online:'));
            console.log(plantUMLViewerURL(newSource));
          } catch (err) {
            console.log(kleur.yellow(`⚠ PNG render failed: ${(err as Error).message}`));
          }
        } else {
          console.log(kleur.dim('  (proposal rejected)'));
        }
      }
    }

    // Optional: open an interactive Socrates conversation
    if (opts.converse) {
      console.log();
      console.log(kleur.dim('─'.repeat(60)));
      console.log(kleur.bold('Conversation with Socrates'));
      console.log(kleur.dim('Type :done or Ctrl+D to end · max 5 turns'));
      console.log(kleur.dim('─'.repeat(60)));

      const conv = await runConversation(seed, gen.model, det.findings, {
        outPath: `${outDir}/conversation.md`,
      });
      console.log();
      console.log(kleur.green('✓ conversation saved'));
      console.log(
        kleur.dim(
          `  ${conv.turns.length} turns · ${conv.durationMs}ms · ${conv.inputTokens} in / ${conv.outputTokens} out`
        )
      );
      console.log(kleur.dim(`  ${outDir}/conversation.md`));
    }
  });

function countFindings(findings: { kind: string }[]): { assumption: number; risk: number; inconsistency: number } {
  const c = { assumption: 0, risk: 0, inconsistency: 0 };
  for (const f of findings) {
    if (f.kind in c) (c as Record<string, number>)[f.kind]++;
  }
  return c;
}

function printTopFindings(findings: import('./types.js').Finding[]): void {
  const byKind = (k: string) => findings.filter(f => f.kind === k).slice(0, 3);
  const sections: Array<[string, string, ReturnType<typeof byKind>]> = [
    ['assumptions', kleur.cyan('●'), byKind('assumption')],
    ['risks', kleur.red('▲'), byKind('risk')],
    ['inconsistencies', kleur.yellow('!'), byKind('inconsistency')],
  ];
  for (const [label, glyph, items] of sections) {
    if (items.length === 0) continue;
    console.log(kleur.bold(`  ${label}:`));
    for (const f of items) {
      const links = f.linkedElementIds.length > 0 ? kleur.dim(` [${f.linkedElementIds.join(', ')}]`) : '';
      const conf = kleur.dim(` (${f.confidence.toFixed(2)})`);
      console.log(`    ${glyph} ${f.text}${links}${conf}`);
    }
  }
}

function truncate(s: string, n: number): string {
  return s.length > n ? s.slice(0, n - 1) + '…' : s;
}

program
  .command('corpus')
  .description('Run all non-holdout seeds through generation + detection; write a report')
  .option('--final', 'include holdout seeds (final-round only; requires PHASE0_FINAL_ROUND=1)')
  .option('--skip-png', 'skip PNG rendering (faster — just .puml + JSON)')
  .option('--score', 'after corpus runs, score every seed with the LLM judge')
  .option('-o, --out <dir>', 'output root (default: ./outputs)')
  .option('-r, --report <file>', 'report path (default: ./reports/<timestamp>-run.md)')
  .action(async (opts: { final?: boolean; skipPng?: boolean; score?: boolean; out?: string; report?: string }) => {
    await runCorpus({
      includeHoldouts: !!opts.final,
      outRoot: opts.out ? resolve(opts.out) : undefined,
      reportPath: opts.report ? resolve(opts.report) : undefined,
      skipPNG: !!opts.skipPng,
      score: !!opts.score,
    });
  });

program
  .command('score <seedId>')
  .description('Score one seed\'s outputs with the LLM judge (writes outputs/<seedId>/score.json)')
  .action(async (seedId: string) => {
    console.log(kleur.dim(`> scoring ${seedId} with ${describeClient()}`));
    const judgeResult = await judgeSeed(seedId);
    const s = judgeResult.score;
    const avg = averageScore(s);
    const min = minDimension(s);
    console.log(kleur.green('✓ scored'));
    console.log(
      kleur.dim(
        `  ${judgeResult.durationMs}ms · ${judgeResult.inputTokens} in / ${judgeResult.outputTokens} out · ` +
          `avg ${avg.toFixed(2)} · min ${min}`
      )
    );
    console.log();
    console.log(kleur.bold('Scores:'));
    const dims: Array<[keyof typeof judgeResult.reasons, string]> = [
      ['modelCoverage', 'model.coverage   '],
      ['modelAccuracy', 'model.accuracy   '],
      ['modelParsimony', 'model.parsimony  '],
      ['constraintCapture', 'constraint.capture'],
      ['assumptionDetectionQuality', 'asm.detection    '],
      ['riskDetectionQuality', 'risk.detection   '],
      ['voiceAndCharacter', 'voice.character  '],
    ];
    for (const [key, label] of dims) {
      const v = (s as unknown as Record<string, number>)[key]!;
      const bar = '█'.repeat(v) + '░'.repeat(5 - v);
      const color = v >= 4 ? kleur.green : v >= 3 ? kleur.yellow : kleur.red;
      console.log(`  ${label}  ${color(bar)} ${v}  ${kleur.dim('— ' + judgeResult.reasons[key])}`);
    }
    if (judgeResult.overallComment) {
      console.log();
      console.log(kleur.bold('Comment:'), judgeResult.overallComment);
    }
  });

program
  .command('config')
  .description('Print resolved configuration')
  .action(() => {
    console.log(kleur.bold('LM Studio'));
    console.log(`  baseURL: ${process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1'}`);
    console.log(`  model:   ${modelName}`);
    console.log(`  apiKey:  ${process.env.LMSTUDIO_API_KEY ? kleur.green('set') : kleur.yellow('default (lm-studio)')}`);
    console.log();
    console.log(kleur.bold('PlantUML'));
    console.log(`  server:  ${process.env.PLANTUML_SERVER ?? 'https://www.plantuml.com/plantuml'}`);
  });

program.parseAsync(process.argv).catch(err => {
  console.error(kleur.red(err instanceof Error ? err.message : String(err)));
  process.exit(1);
});