Initial commit — design docs + Phase 0 validation harness
Sets up the Socrata project repo with: docs/ — strategy and design documents - idea.md: full product vision - implementation-plan.md: Phase 0 + Phase 1 MVP plan - phase-0-validation.md: 2-week validation experiment strategy - phase-0-plan.md: concrete Phase 0 build plan - phase-0-results.md: Phase 0 gate outcome — GO for MVP - sysml-modeling.md: metamodel + SE discipline + validation rules - socrates.md: agent character, surfaces, modes, prompts, lifecycle - sync.md: bidirectional text↔diagram sync engineering - design-source/: HTML/CSS/JS handoff bundle from Claude Design phase-0/ — validated harness (CLI, no UI, no DB) - LM Studio (local OpenAI-compatible) generation + detection + judge - PlantUML rendering for SysML model visualization - 10-seed corpus (8 working + 2 holdouts) - 5 corpus runs with iteration history in reports/ - Final gate: 10/10 pass, mean 4.32/5, holdouts validated Phase 1 MVP scope and milestones documented in implementation-plan.md.
This commit is contained in:
12
phase-0/.env.example
Normal file
12
phase-0/.env.example
Normal file
@@ -0,0 +1,12 @@
|
||||
# LM Studio configuration
|
||||
LMSTUDIO_BASE_URL=http://localhost:1234/v1
|
||||
LMSTUDIO_API_KEY=lm-studio
|
||||
LMSTUDIO_MODEL=google/gemma-4-e4b
|
||||
|
||||
# PlantUML rendering — public server by default
|
||||
PLANTUML_SERVER=https://www.plantuml.com/plantuml
|
||||
|
||||
# Generation defaults
|
||||
PHASE0_TEMP_GENERATE=0.3
|
||||
PHASE0_TEMP_DETECT=0.2
|
||||
PHASE0_TEMP_REVIEW=0.4
|
||||
7
phase-0/.gitignore
vendored
Normal file
7
phase-0/.gitignore
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
node_modules/
|
||||
dist/
|
||||
outputs/
|
||||
.env
|
||||
.env.local
|
||||
*.log
|
||||
.DS_Store
|
||||
41
phase-0/README.md
Normal file
41
phase-0/README.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Socrata · Phase 0 Validation Harness
|
||||
|
||||
Validation experiment for the Socrata project. Runs a corpus of seed ideas through:
|
||||
seed → SysML model generation → PlantUML render → Socrates analysis (assumptions, risks, inconsistencies) → conversation → scoring.
|
||||
|
||||
See [../docs/phase-0-validation.md](../docs/phase-0-validation.md) and [../docs/phase-0-plan.md](../docs/phase-0-plan.md) for context.
|
||||
|
||||
## Setup
|
||||
|
||||
```bash
|
||||
pnpm install
|
||||
cp .env.example .env.local # then fill in your LM Studio config
|
||||
```
|
||||
|
||||
LM Studio must be running locally with a model loaded. Default endpoint: `http://localhost:1234/v1`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
pnpm phase0 hello # smoke test — round-trip a chat to LM Studio
|
||||
pnpm phase0 render-test # render the Aristotle reference model as PlantUML PNG
|
||||
pnpm phase0 run <seed-id> # full pipeline on one seed
|
||||
pnpm phase0 corpus # all seeds (excluding holdouts)
|
||||
```
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
src/
|
||||
cli.ts entrypoint
|
||||
types.ts shared types
|
||||
llm/ LM Studio client + messages helper
|
||||
generate/ seed → SysML model
|
||||
socrates/ detect / converse / propose
|
||||
render/ PlantUML serialization + rendering
|
||||
eval/ rubric scoring + reports
|
||||
prompts/ versioned prompt files (.md)
|
||||
seeds/ corpus of test seed ideas (JSON)
|
||||
outputs/ generated artifacts (gitignored)
|
||||
reports/ committed run reports
|
||||
```
|
||||
30
phase-0/package.json
Normal file
30
phase-0/package.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "socrata-phase-0",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"description": "Phase 0 validation harness — local LM Studio + PlantUML",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"phase0": "./dist/cli.js"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"dev": "tsx src/cli.ts",
|
||||
"phase0": "tsx src/cli.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"openai": "^4.79.0",
|
||||
"zod": "^3.24.1",
|
||||
"commander": "^13.0.0",
|
||||
"plantuml-encoder": "^1.4.0",
|
||||
"dotenv": "^16.4.7",
|
||||
"kleur": "^4.1.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.10.5",
|
||||
"@types/plantuml-encoder": "^1.4.2",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^5.7.2"
|
||||
}
|
||||
}
|
||||
697
phase-0/pnpm-lock.yaml
generated
Normal file
697
phase-0/pnpm-lock.yaml
generated
Normal file
@@ -0,0 +1,697 @@
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
commander:
|
||||
specifier: ^13.0.0
|
||||
version: 13.1.0
|
||||
dotenv:
|
||||
specifier: ^16.4.7
|
||||
version: 16.6.1
|
||||
kleur:
|
||||
specifier: ^4.1.5
|
||||
version: 4.1.5
|
||||
openai:
|
||||
specifier: ^4.79.0
|
||||
version: 4.104.0(zod@3.25.76)
|
||||
plantuml-encoder:
|
||||
specifier: ^1.4.0
|
||||
version: 1.4.0
|
||||
zod:
|
||||
specifier: ^3.24.1
|
||||
version: 3.25.76
|
||||
devDependencies:
|
||||
'@types/node':
|
||||
specifier: ^22.10.5
|
||||
version: 22.19.17
|
||||
'@types/plantuml-encoder':
|
||||
specifier: ^1.4.2
|
||||
version: 1.4.2
|
||||
tsx:
|
||||
specifier: ^4.19.2
|
||||
version: 4.21.0
|
||||
typescript:
|
||||
specifier: ^5.7.2
|
||||
version: 5.9.3
|
||||
|
||||
packages:
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.7':
|
||||
resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ppc64]
|
||||
os: [aix]
|
||||
|
||||
'@esbuild/android-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/android-arm@0.27.7':
|
||||
resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/android-x64@0.27.7':
|
||||
resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [android]
|
||||
|
||||
'@esbuild/darwin-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [darwin]
|
||||
|
||||
'@esbuild/darwin-x64@0.27.7':
|
||||
resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [darwin]
|
||||
|
||||
'@esbuild/freebsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [freebsd]
|
||||
|
||||
'@esbuild/freebsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [freebsd]
|
||||
|
||||
'@esbuild/linux-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-arm@0.27.7':
|
||||
resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-ia32@0.27.7':
|
||||
resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ia32]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-loong64@0.27.7':
|
||||
resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [loong64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-mips64el@0.27.7':
|
||||
resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [mips64el]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-ppc64@0.27.7':
|
||||
resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ppc64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-riscv64@0.27.7':
|
||||
resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [riscv64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-s390x@0.27.7':
|
||||
resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [s390x]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/linux-x64@0.27.7':
|
||||
resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [linux]
|
||||
|
||||
'@esbuild/netbsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [netbsd]
|
||||
|
||||
'@esbuild/netbsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [netbsd]
|
||||
|
||||
'@esbuild/openbsd-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [openbsd]
|
||||
|
||||
'@esbuild/openbsd-x64@0.27.7':
|
||||
resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [openbsd]
|
||||
|
||||
'@esbuild/openharmony-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [openharmony]
|
||||
|
||||
'@esbuild/sunos-x64@0.27.7':
|
||||
resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [sunos]
|
||||
|
||||
'@esbuild/win32-arm64@0.27.7':
|
||||
resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [arm64]
|
||||
os: [win32]
|
||||
|
||||
'@esbuild/win32-ia32@0.27.7':
|
||||
resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [ia32]
|
||||
os: [win32]
|
||||
|
||||
'@esbuild/win32-x64@0.27.7':
|
||||
resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==}
|
||||
engines: {node: '>=18'}
|
||||
cpu: [x64]
|
||||
os: [win32]
|
||||
|
||||
'@types/node-fetch@2.6.13':
|
||||
resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==}
|
||||
|
||||
'@types/node@18.19.130':
|
||||
resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==}
|
||||
|
||||
'@types/node@22.19.17':
|
||||
resolution: {integrity: sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==}
|
||||
|
||||
'@types/plantuml-encoder@1.4.2':
|
||||
resolution: {integrity: sha512-R60SYlcXyTXQebyVwcarGSJUucuwYY5wetBWAOnRt3k52flBXgt4R1KrxaZ4xpk5MpVo9titpVBBhgl/A9JWRQ==}
|
||||
|
||||
abort-controller@3.0.0:
|
||||
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
|
||||
engines: {node: '>=6.5'}
|
||||
|
||||
agentkeepalive@4.6.0:
|
||||
resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==}
|
||||
engines: {node: '>= 8.0.0'}
|
||||
|
||||
asynckit@0.4.0:
|
||||
resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
|
||||
|
||||
call-bind-apply-helpers@1.0.2:
|
||||
resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
combined-stream@1.0.8:
|
||||
resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
|
||||
engines: {node: '>= 0.8'}
|
||||
|
||||
commander@13.1.0:
|
||||
resolution: {integrity: sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
delayed-stream@1.0.0:
|
||||
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
|
||||
engines: {node: '>=0.4.0'}
|
||||
|
||||
dotenv@16.6.1:
|
||||
resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
dunder-proto@1.0.1:
|
||||
resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
es-define-property@1.0.1:
|
||||
resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
es-errors@1.3.0:
|
||||
resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
es-object-atoms@1.1.1:
|
||||
resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
es-set-tostringtag@2.1.0:
|
||||
resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
esbuild@0.27.7:
|
||||
resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==}
|
||||
engines: {node: '>=18'}
|
||||
hasBin: true
|
||||
|
||||
event-target-shim@5.0.1:
|
||||
resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
form-data-encoder@1.7.2:
|
||||
resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}
|
||||
|
||||
form-data@4.0.5:
|
||||
resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
formdata-node@4.4.1:
|
||||
resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}
|
||||
engines: {node: '>= 12.20'}
|
||||
|
||||
fsevents@2.3.3:
|
||||
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
|
||||
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
|
||||
os: [darwin]
|
||||
|
||||
function-bind@1.1.2:
|
||||
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
|
||||
|
||||
get-intrinsic@1.3.0:
|
||||
resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
get-proto@1.0.1:
|
||||
resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
get-tsconfig@4.14.0:
|
||||
resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==}
|
||||
|
||||
gopd@1.2.0:
|
||||
resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
has-symbols@1.1.0:
|
||||
resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
has-tostringtag@1.0.2:
|
||||
resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
hasown@2.0.3:
|
||||
resolution: {integrity: sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
humanize-ms@1.2.1:
|
||||
resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==}
|
||||
|
||||
kleur@4.1.5:
|
||||
resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
math-intrinsics@1.1.0:
|
||||
resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
|
||||
engines: {node: '>= 0.4'}
|
||||
|
||||
mime-db@1.52.0:
|
||||
resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
|
||||
engines: {node: '>= 0.6'}
|
||||
|
||||
mime-types@2.1.35:
|
||||
resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
|
||||
engines: {node: '>= 0.6'}
|
||||
|
||||
ms@2.1.3:
|
||||
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
|
||||
|
||||
node-domexception@1.0.0:
|
||||
resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
|
||||
engines: {node: '>=10.5.0'}
|
||||
deprecated: Use your platform's native DOMException instead
|
||||
|
||||
node-fetch@2.7.0:
|
||||
resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
|
||||
engines: {node: 4.x || >=6.0.0}
|
||||
peerDependencies:
|
||||
encoding: ^0.1.0
|
||||
peerDependenciesMeta:
|
||||
encoding:
|
||||
optional: true
|
||||
|
||||
openai@4.104.0:
|
||||
resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
ws: ^8.18.0
|
||||
zod: ^3.23.8
|
||||
peerDependenciesMeta:
|
||||
ws:
|
||||
optional: true
|
||||
zod:
|
||||
optional: true
|
||||
|
||||
plantuml-encoder@1.4.0:
|
||||
resolution: {integrity: sha512-sxMwpDw/ySY1WB2CE3+IdMuEcWibJ72DDOsXLkSmEaSzwEUaYBT6DWgOfBiHGCux4q433X6+OEFWjlVqp7gL6g==}
|
||||
|
||||
resolve-pkg-maps@1.0.0:
|
||||
resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
|
||||
|
||||
tr46@0.0.3:
|
||||
resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
|
||||
|
||||
tsx@4.21.0:
|
||||
resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
hasBin: true
|
||||
|
||||
typescript@5.9.3:
|
||||
resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
|
||||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
undici-types@5.26.5:
|
||||
resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
|
||||
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
web-streams-polyfill@4.0.0-beta.3:
|
||||
resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==}
|
||||
engines: {node: '>= 14'}
|
||||
|
||||
webidl-conversions@3.0.1:
|
||||
resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
|
||||
|
||||
whatwg-url@5.0.0:
|
||||
resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
|
||||
|
||||
zod@3.25.76:
|
||||
resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==}
|
||||
|
||||
snapshots:
|
||||
|
||||
'@esbuild/aix-ppc64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-arm@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/android-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/darwin-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/darwin-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/freebsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/freebsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-arm@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-ia32@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-loong64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-mips64el@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-ppc64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-riscv64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-s390x@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/linux-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/netbsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/netbsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openbsd-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openbsd-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/openharmony-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/sunos-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-arm64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-ia32@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@esbuild/win32-x64@0.27.7':
|
||||
optional: true
|
||||
|
||||
'@types/node-fetch@2.6.13':
|
||||
dependencies:
|
||||
'@types/node': 22.19.17
|
||||
form-data: 4.0.5
|
||||
|
||||
'@types/node@18.19.130':
|
||||
dependencies:
|
||||
undici-types: 5.26.5
|
||||
|
||||
'@types/node@22.19.17':
|
||||
dependencies:
|
||||
undici-types: 6.21.0
|
||||
|
||||
'@types/plantuml-encoder@1.4.2':
|
||||
dependencies:
|
||||
'@types/node': 22.19.17
|
||||
|
||||
abort-controller@3.0.0:
|
||||
dependencies:
|
||||
event-target-shim: 5.0.1
|
||||
|
||||
agentkeepalive@4.6.0:
|
||||
dependencies:
|
||||
humanize-ms: 1.2.1
|
||||
|
||||
asynckit@0.4.0: {}
|
||||
|
||||
call-bind-apply-helpers@1.0.2:
|
||||
dependencies:
|
||||
es-errors: 1.3.0
|
||||
function-bind: 1.1.2
|
||||
|
||||
combined-stream@1.0.8:
|
||||
dependencies:
|
||||
delayed-stream: 1.0.0
|
||||
|
||||
commander@13.1.0: {}
|
||||
|
||||
delayed-stream@1.0.0: {}
|
||||
|
||||
dotenv@16.6.1: {}
|
||||
|
||||
dunder-proto@1.0.1:
|
||||
dependencies:
|
||||
call-bind-apply-helpers: 1.0.2
|
||||
es-errors: 1.3.0
|
||||
gopd: 1.2.0
|
||||
|
||||
es-define-property@1.0.1: {}
|
||||
|
||||
es-errors@1.3.0: {}
|
||||
|
||||
es-object-atoms@1.1.1:
|
||||
dependencies:
|
||||
es-errors: 1.3.0
|
||||
|
||||
es-set-tostringtag@2.1.0:
|
||||
dependencies:
|
||||
es-errors: 1.3.0
|
||||
get-intrinsic: 1.3.0
|
||||
has-tostringtag: 1.0.2
|
||||
hasown: 2.0.3
|
||||
|
||||
esbuild@0.27.7:
|
||||
optionalDependencies:
|
||||
'@esbuild/aix-ppc64': 0.27.7
|
||||
'@esbuild/android-arm': 0.27.7
|
||||
'@esbuild/android-arm64': 0.27.7
|
||||
'@esbuild/android-x64': 0.27.7
|
||||
'@esbuild/darwin-arm64': 0.27.7
|
||||
'@esbuild/darwin-x64': 0.27.7
|
||||
'@esbuild/freebsd-arm64': 0.27.7
|
||||
'@esbuild/freebsd-x64': 0.27.7
|
||||
'@esbuild/linux-arm': 0.27.7
|
||||
'@esbuild/linux-arm64': 0.27.7
|
||||
'@esbuild/linux-ia32': 0.27.7
|
||||
'@esbuild/linux-loong64': 0.27.7
|
||||
'@esbuild/linux-mips64el': 0.27.7
|
||||
'@esbuild/linux-ppc64': 0.27.7
|
||||
'@esbuild/linux-riscv64': 0.27.7
|
||||
'@esbuild/linux-s390x': 0.27.7
|
||||
'@esbuild/linux-x64': 0.27.7
|
||||
'@esbuild/netbsd-arm64': 0.27.7
|
||||
'@esbuild/netbsd-x64': 0.27.7
|
||||
'@esbuild/openbsd-arm64': 0.27.7
|
||||
'@esbuild/openbsd-x64': 0.27.7
|
||||
'@esbuild/openharmony-arm64': 0.27.7
|
||||
'@esbuild/sunos-x64': 0.27.7
|
||||
'@esbuild/win32-arm64': 0.27.7
|
||||
'@esbuild/win32-ia32': 0.27.7
|
||||
'@esbuild/win32-x64': 0.27.7
|
||||
|
||||
event-target-shim@5.0.1: {}
|
||||
|
||||
form-data-encoder@1.7.2: {}
|
||||
|
||||
form-data@4.0.5:
|
||||
dependencies:
|
||||
asynckit: 0.4.0
|
||||
combined-stream: 1.0.8
|
||||
es-set-tostringtag: 2.1.0
|
||||
hasown: 2.0.3
|
||||
mime-types: 2.1.35
|
||||
|
||||
formdata-node@4.4.1:
|
||||
dependencies:
|
||||
node-domexception: 1.0.0
|
||||
web-streams-polyfill: 4.0.0-beta.3
|
||||
|
||||
fsevents@2.3.3:
|
||||
optional: true
|
||||
|
||||
function-bind@1.1.2: {}
|
||||
|
||||
get-intrinsic@1.3.0:
|
||||
dependencies:
|
||||
call-bind-apply-helpers: 1.0.2
|
||||
es-define-property: 1.0.1
|
||||
es-errors: 1.3.0
|
||||
es-object-atoms: 1.1.1
|
||||
function-bind: 1.1.2
|
||||
get-proto: 1.0.1
|
||||
gopd: 1.2.0
|
||||
has-symbols: 1.1.0
|
||||
hasown: 2.0.3
|
||||
math-intrinsics: 1.1.0
|
||||
|
||||
get-proto@1.0.1:
|
||||
dependencies:
|
||||
dunder-proto: 1.0.1
|
||||
es-object-atoms: 1.1.1
|
||||
|
||||
get-tsconfig@4.14.0:
|
||||
dependencies:
|
||||
resolve-pkg-maps: 1.0.0
|
||||
|
||||
gopd@1.2.0: {}
|
||||
|
||||
has-symbols@1.1.0: {}
|
||||
|
||||
has-tostringtag@1.0.2:
|
||||
dependencies:
|
||||
has-symbols: 1.1.0
|
||||
|
||||
hasown@2.0.3:
|
||||
dependencies:
|
||||
function-bind: 1.1.2
|
||||
|
||||
humanize-ms@1.2.1:
|
||||
dependencies:
|
||||
ms: 2.1.3
|
||||
|
||||
kleur@4.1.5: {}
|
||||
|
||||
math-intrinsics@1.1.0: {}
|
||||
|
||||
mime-db@1.52.0: {}
|
||||
|
||||
mime-types@2.1.35:
|
||||
dependencies:
|
||||
mime-db: 1.52.0
|
||||
|
||||
ms@2.1.3: {}
|
||||
|
||||
node-domexception@1.0.0: {}
|
||||
|
||||
node-fetch@2.7.0:
|
||||
dependencies:
|
||||
whatwg-url: 5.0.0
|
||||
|
||||
openai@4.104.0(zod@3.25.76):
|
||||
dependencies:
|
||||
'@types/node': 18.19.130
|
||||
'@types/node-fetch': 2.6.13
|
||||
abort-controller: 3.0.0
|
||||
agentkeepalive: 4.6.0
|
||||
form-data-encoder: 1.7.2
|
||||
formdata-node: 4.4.1
|
||||
node-fetch: 2.7.0
|
||||
optionalDependencies:
|
||||
zod: 3.25.76
|
||||
transitivePeerDependencies:
|
||||
- encoding
|
||||
|
||||
plantuml-encoder@1.4.0: {}
|
||||
|
||||
resolve-pkg-maps@1.0.0: {}
|
||||
|
||||
tr46@0.0.3: {}
|
||||
|
||||
tsx@4.21.0:
|
||||
dependencies:
|
||||
esbuild: 0.27.7
|
||||
get-tsconfig: 4.14.0
|
||||
optionalDependencies:
|
||||
fsevents: 2.3.3
|
||||
|
||||
typescript@5.9.3: {}
|
||||
|
||||
undici-types@5.26.5: {}
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
web-streams-polyfill@4.0.0-beta.3: {}
|
||||
|
||||
webidl-conversions@3.0.1: {}
|
||||
|
||||
whatwg-url@5.0.0:
|
||||
dependencies:
|
||||
tr46: 0.0.3
|
||||
webidl-conversions: 3.0.1
|
||||
|
||||
zod@3.25.76: {}
|
||||
295
phase-0/reports/2026-04-28-1735-run.md
Normal file
295
phase-0/reports/2026-04-28-1735-run.md
Normal file
@@ -0,0 +1,295 @@
|
||||
# Phase 0 corpus run · 2026-04-28T15:35:14.522Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 8 · **OK:** 8 · **Failed:** 0
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks | Assoc | Constr | Req | Asm | Risk | Inco | Lint | Stripped | Conf | Gen (s) | Det (s) | Status |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 5 | 3 | 3 | 3 | 1 | 3 | 3 | 0 | 4 | 0.90 | 24.0 | 20.2 | ✓ |
|
||||
| 02-habit-coach | medium | 6 | 6 | 3 | 3 | 3 | 4 | 3 | 0 | 0 | 0.90 | 34.1 | 26.7 | ✓ |
|
||||
| 03-redline-ai | medium | 4 | 3 | 4 | 4 | 5 | 3 | 3 | 0 | 0 | 0.95 | 21.7 | 28.4 | ✓ |
|
||||
| 04-skillswap | medium | 5 | 5 | 3 | 3 | 3 | 5 | 3 | 0 | 0 | 0.90 | 31.8 | 28.5 | ✓ |
|
||||
| 05-cyclist-thing | failure-prone | 2 | 1 | 0 | 0 | 1 | 3 | 1 | 0 | 0 | 0.30 | 7.1 | 11.5 | ✓ |
|
||||
| 06-eventstream | hard | 5 | 4 | 3 | 4 | 4 | 4 | 3 | 0 | 0 | 0.90 | 33.7 | 25.8 | ✓ |
|
||||
| 07-pet-translator | failure-prone | 4 | 3 | 2 | 3 | 4 | 4 | 3 | 0 | 0 | 0.85 | 34.7 | 32.5 | ✓ |
|
||||
| 08-quiet-hours | medium | 6 | 5 | 3 | 3 | 4 | 3 | 4 | 0 | 0 | 0.95 | 49.0 | 32.6 | ✓ |
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/dLNRRXen47tVhnYHbnOLHCX5QOIYf56YFFIIfkhpohix2ovUU-knGIX2waVq2_ib7U-5B0b4Qbz0p8pFEJDpR2x92UjyeIAQIrqAAmf8H3gVMkDrTcsKiN2Gds2Euqu5pKHcbb9F8HUAi9D9bI22nmZWmmkKB1VvA6ICpT0sqUHCl3yRS_JAMhDiWgTuaXrb7BmrsdqM1KBl3jK2dKn5Brft-BHnk8TkX4VfyHwwTJIJ6G8MfLj1WKM2bgAe5j-xifASSGfx80X4-mic4rgHm-BY0Wx6wR4uJIfcgSb9vvqqMgWu2HA4NS4v9COe59fBb5d6FEWvKYorGolIKDqfMRVStpLhdLgHUS6DVNC-G-qg7RwRORGgmveOorYM5AIanbl2fR7pkE0gAuLY4dAMLyNfJ1BggPXYmKTZxck-IyBr1WG-DI2LW5tWaqcYJ3flwCadFp1r8StvBdFGZLcOWVF42TI-UFIw3h-hlzPF4ZuoYhS8rxp-jdzLn-AqZZsZTgQKQPUHM6sAEyQm8VcIvp95k0jjLGoc2iIp47W6BnP2Af589TrgQsOjs2TF3ZGkk6tZNUaT21vOKIhaDP9HbJrO9OVhRg6q9b5OLCm5duwhqp7NBeBLk4qMBDWkbOIDwWUaabC8YZlJwQhs1HmEnmJt9wD0KLeaj8oIi-_gsTMrSSa5-w5lRnxkhu0hHS3zy-jtP-LDRzpr0cdRyj1QlgBAqPOYRa5sa4MxZJmON7IksZcqzuJYfNIpw2aL2dVxd2UQ8X7IAnIxJuRV51iTlLZ4SBWjIvIbadoDdOcwGtsbR7iW_rIuMspdIAHmYZe3AwSpnpRjzn_mfvUskk3KxzUXcw-3qUYmtwzMsZmf11O7zMC0OS2m8LX8mIPbvmpg5mlSJEfvUElvnNMQJH1WmXasUfXvvjEGRKrEmmxhk67zx-lH1Jjgm3eFLDWNTrAmlof9_ELckCUVyAPo-DjXn3Eh9nTTyWV_-_q5)
|
||||
- **Notes:** Aristotle is the SoI; Undergraduate Student is the primary actor; Coursework Material is the core block being processed.
|
||||
- **Post-validate:** stripped 4 refs, dropped 2 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Undergraduate Student (actor), Coursework Material (block), Lecture Content (block), Office Hours (block)
|
||||
- Associations: undergraduate_student→aristotle:interacts_with, aristotle→coursework_material:processes, undergraduate_student→coursework_material:works_on
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.75) The design assumes that 'low-stakes' interaction will be sufficient to re-engage students with coursework material. [req_001]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The requirement for FERPA tenancy and processing coursework material locally may introduce significant technical complexity or limit model capability if the underlying infrastructure cannot guarantee data residency. [ferpa_tenancy, aristotle, coursework_material]
|
||||
- (0.80 · medium) The combination of a low-stakes interaction requirement and the constraint to never solve problems risks creating an ambiguous user experience where students are unsure how to prompt for help without crossing the boundary into seeking solutions. [low_stakes_interaction, must_never_solve, req_001]
|
||||
- (0.70 · medium) Achieving a P50 latency of 1.2s while processing complex coursework material and maintaining FERPA tenancy may require specialized, high-cost infrastructure that impacts scalability. [latency_p50, aristotle]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association `a2` states that `aristotle` processes `coursework_material`, but the system's core function is defined by `req_001` as providing a low-stakes thinking partner, which suggests an interaction with the student or material rather than just processing the material itself. [a2, aristotle, coursework_material]
|
||||
- (0.75) The requirement `req_001` specifies that the thinking partner helps students re-engage without producing solutions, but this goal is not explicitly linked to any specific block or association that defines the mechanism of 'low-stakes interaction' beyond merely satisfying it with the `aristotle` system. [req_001, aristotle]
|
||||
- (0.80) The block `office_hours` is defined as a resource the student interacts with in the seed payload's context (the 'long tail between lectures and office hours'), but it has no associations or requirements linking it to the system, making its inclusion potentially unused. [office_hours]
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLRRSjis47tdL-muBwqcSbpdCgdhySIruoJJD57Zz1cp19OaAX1W0L2gcl5CFw9Vs2_f0gGi-leWpH1OxfuzUyvARq947_lM566fROSUMoXHBcllUglEd74UdbGlgQB3IKHeKBcrjZLKQ09DRgJ14E1R0V3pdIogmkeWtJYlo8-dvMlyyVKXdvvwxzRZuIjwgPuhFhnmDdx2bc3lFPaLHIrnhxYQrBCkqWFb3kYvVF50kQkYqOg0sYvkeAMs91-AOW2_znvB7U7CeMps00CqwLd8z0p7ns4J8hKd9_3aKBx0LsMkhZ2YmBhsL6FKpWgojRO4Hr0wPmWjnp1Wib4e34tfqAl9vTMsz2B7vAf3E1T46PtdUgaEIiajCXXFYcyriyzfGlGy2xwMZJPATDvLsj3kuZhzMRhFsNFahUJIsP0G6bgHuRV9zkst4eEMJr6jq4fILxbNRJQY3rYJI7tVK-USLbeIVBw4qyM7N4_b4-62m4vptT8ukHphQZuDt15P4NLxdNJ7NErSRKXKsARAHjjbpd8FVxydL73E2NBL2SYR9R7hXCH8jVERAO6gzyFyMctxI8bS9e7LmNS96xkbxUuZrjaLIoY_2nzTFN0yEHJ6rRSmHDTfUQCu0vhclax-UR4uVVRkVF5b_WlBiksCJiCOQtGTYbfrNYn5cUIFtD7nSPfbzAXj7AlbqMUTGjk722MD2fjpQFJE65A0DdsStRJwRtwis37yDASG1h51HHXxJs5_rpeF-ODb9WVYs6q3XeoE1wjtTu_Y-pM1MZk_13JErY7PKtFkoomC-ChaPtZliXRW8ci1Fh8MVWA9blS0UEeCiknYG-q4u2S7bqucz1SZ-epDEX7oiU1jbXY_3zNNXc0m_C0QMwlHQJkmx7YfP1zpcnwwlZGypMqgLmtQouGdBCLev_dyvCO-EO8MBGUoBqPhhpGMWpllnYOoFLisYBMEJN7JL2dyXYih8XM4nxLeYJKbWuYkc2QSpUVRDNF4hL6QSjWjgDlXgS20x0Xgieo47S833hfk-3rMoxRBbFYE-ezuG2jDwo0Q7NZhR8IhYllrkxz_ktCshD6KEtX8L8--T7VQZuONXhX31JwrnZgOpRxGdxtsr2QAPhFXwErlyuE37sOp-E-VVs7XtIg9DkKdzHHS-GV9g5U31hANraHB5ktMTvdAAKrigrtUmp7lwVYhGi2J75KrpXCgxrgWlvZErD6mRWDynxvvDlZa-sd6vsF6jpQm2zXdPe3MiAzCIY3pRk6jbzG5UUqCdXjM3yFepHOeipDmLBpXB_u3yJy0)
|
||||
- **Notes:** The Habit Coach is the SoI; Parent and Child are actors; OS APIs and Usage Data are key blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (actor), Device OS API (block), Usage Data (block), Conversation Log (block)
|
||||
- Associations: parent→habit_coach:manages_account_via, child→habit_coach:interacts_with, device_os_api→usage_data:feeds_raw_metrics_to, habit_coach→parent:presents_dashboard_to, habit_coach→child:generates_insights_for, parent→conversation_log:reviews_history_of
|
||||
- Constraints: COPPA/GDPR-K Compliance, OS Integration Boundary, No Social Features
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The system assumes that parents will value objective data (req_001) enough to initiate and sustain a weekly conversation, rather than simply using the tool for conflict avoidance.
|
||||
- (0.85) It is assumed that existing OS APIs (device_os_api) will provide sufficient granularity of data (usage_data) to move beyond simple friction reporting and support meaningful, objective conversation topics.
|
||||
- (0.75) The model assumes the child's consent level (child_profile) is a reliable input that can be managed by the parent without creating new points of conflict or resistance.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system cannot reliably generate the shared, objective data required for weekly conversations if it relies solely on raw metrics from external APIs (device_os_api) without a mechanism to interpret or contextualize that usage data. [a3, req_001, usage_data, device_os_api]
|
||||
- (0.85 · medium) The model does not specify how the 'child' actor gains or provides the necessary data (daily_usage_data) to feed into the system, creating a potential gap in the core usage metrics. [child, a2, daily_usage_data]
|
||||
- (0.70 · medium) The constraint 'No Social Features' (no_social_sharing) is too broad and does not account for the necessary data sharing mechanism required to populate the parent dashboard (parent_dashboard), potentially limiting core functionality. [habit_coach, no_social_sharing, a4]
|
||||
- (0.80 · high) The model lacks a defined process for how the 'parent' manages or verifies the child's consent level (consent_level) in compliance with COPPA/GDPR-K, which is critical for data handling. [parent, child, consent_level, coppa_gdpr_k_boundary]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The requirement to provide shared, objective data for weekly parent-child conversations (REQ-001) is not fully supported by the available blocks; specifically, there is no explicit mechanism linking 'conversation_log' or 'usage_data' to both 'parent' and 'child' in a shared context. [req_001, conversation_log, parent, child]
|
||||
- (0.80) The association 'a5' (habit_coach to child: generates_insights_for) is too vague and does not specify the nature of the insights, potentially over-promising functionality beyond what is defined by the existing blocks. [a5, habit_coach, child]
|
||||
- (0.75) The association 'a6' (parent to conversation_log: reviews_history_of) implies a one-way review process, which contradicts the desired outcome of a shared parent-child conversation. [a6, parent, conversation_log]
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.95
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/fLNRJXin47tVhnZ1InirY8QBQ8GGb9OM2LgL8lNHclNE9Yw-R6rlqWWXzIFwXVsIZdSJiWH27_goaiVZSyvSzpX4zB4sEWitobReqK2EycRaNMsBKwUTXyroZqeQT3p260itLNO49Uf0dHkfCGIupG3UFa8fIYotqutp1VcvDJ_0DmS3jfvuxwPpupxj5Ji56y-SZP_G46ny93sXg2HkP7STFkiYhQ7RfXsvkuRkBXkhWe1C5MTWoEJaGvQruZUkgD3A4foSRm06yErHe8B3mp0BaSpH4MmEv2xkvmstif567gDoLbGQO-cyWI64w3b5VC-uTI2XLUxHpmHAISmpXDmvJMXPpPpvWaQeuPhGDCmw7KNa8pEZZCupSIBq_8xVawtDxOWiUNQJ77iW_UfxRPNpWl6SL1ZfRWb_ojdqZ0FldAmDsTYmo8Nry33NJjxCEOgvZuYpYXv4m_viNvgM-4sSSD76sO8_27s5GZfJePs9rgcR4O09LuJpsMLPq5pM8Txdd62guXWkbVGkk3B2DovtGstOJToxYUGcfdp98ZW4HaM-R9GyJj0YneG1bbWCDFwGM49VijaPDXfNa0xmel9gehXSb5vAyZQyR3HO9veNOl52D31fSjQhEAaB5Hi-3Qw4n3u3EKRByA4UZIYaJckrvTp4Fold8-QQ6ihbKM024YtGUhgldqzX0DUfokVdJKsqGWu2IkyC53W3PrjexgY1w3WyWPbrUhRVF-eEpn1IaREEfTVlBxjl29LtQJ0U-YIGbIOQCcf9dbX2oDPMUMjhXHqhrar1H9SzKvR_UDO9fVDeJJsUurdDytE-AU5a2_1gD8uyDxtU5VsebQSqkw7NQqtll_Itjr_tUl3drsyuKpx4VfMAFqSDN6AUCxG5SkOMSRGZr7O4yQ1n-njbwqXXguCwcADUZma-jBi8JjjTnFy9LslMHXD5KzwddVOD3CUh58lKubZ7iKjJlXJKD5-GN5GvxjBjpEd-lGI0rp14LiMzuu5xugRKRjh5skr8RnTx6t75WjFqFpNiS5_oEgIlN5aPMnbNewr9TimV_hN-1G00)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Clause Library (block)
|
||||
- Associations: legal_team→redline_ai:uses, redline_ai→contract:processes, redline_ai→clause_library:references
|
||||
- Constraints: Must integrate with Microsoft Word, Contract content must never train cross-customer models (privilege concerns), Audit trail of every change suggestion must be exportable, SOC 2 Type II compliance from day one
|
||||
- Requirements: REQ-001, REQ-002, REQ-003, REQ-004
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) Legal teams will adopt a tool that automates mechanical redlining, provided the output is perceived as better than a tired junior associate. [redline_ai, legal_team, req_001]
|
||||
- (0.90) The system's ability to generate redlines must be perceived as a significant improvement over the current manual process performed by junior associates. [redline_ai, legal_team]
|
||||
- (0.85) The General Counsel will trust AI suggestions enough to review them instead of authoring redlines from scratch. [redline_ai, req_002]
|
||||
- (0.80) The system must be able to consistently and accurately apply company-specific clause libraries during the redlining process. [redline_ai, clause_library]
|
||||
- (0.75) The target user base (GCs/Legal Ops) has the operational capacity to handle 100+ contracts per quarter. [redline_ai, legal_team]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Relying on a Microsoft Word integration (must_integrate_with_word) creates a single point of failure for the core user workflow, potentially limiting adoption if MS Office updates break functionality. [redline_ai, must_integrate_with_word, req_003]
|
||||
- (0.85 · high) The requirement for SOC 2 Type II compliance (soc2_compliance) and the need to prevent cross-customer training (no_cross_customer_training) introduce significant overhead in data governance, which could delay time-to-market. [redline_ai, soc2_compliance, no_cross_customer_training, req_004]
|
||||
- (0.95 · high) The system's ability to provide an exportable audit trail (exportable_audit_trail) must be architected into the core data model, otherwise, meeting legal compliance requirements will be impossible. [redline_ai, exportable_audit_trail]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-003 states that the system must integrate with Microsoft Word, but no block or association defines how the `legal_team` interacts with this integration point. [req_003, legal_team, redline_ai]
|
||||
- (0.85) The `clause_library` block is defined with properties like `company_clauses` and `version`, but no requirement or association mandates its use in the redlining process. [clause_library, a3]
|
||||
- (0.75) The `contract` block has a property of `content`, but no requirement or association specifies how this content is processed by the system, beyond the general composition link `a2`. [contract, a2]
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNRRXit47tVhnWuBwqW6QwT1AaG6EaL3T06hLsWZuiHERiQY9SjoPKY10Rw4Vt2V4c6NDbTMIkWBn9mICuvCtDczbrC659lJHKtx3eCQ665QjC6tplzWpS-m8lc5JLqFReHrwZzZbqB3Pf8enDbC4RuL05yVn95DzXSvHCVD8K3kdg3txwv5lIx4Fpk0BwcL_f62_YpT-a3MeABNyXiAR72Y-fXnETyeZDqLtIZNfwXUwZMh0d8TcaFbkoAGgogGVp5_OQDkTzX1xyRJ8qFzW8mGinm5BZk3Z2yVHltCP6zlOKNr-eblbuLBNsaK7V1Dsm8bX1Ja589NYBKxBRaaWzxEIBNsq-Q8hTkRgX5Czy5Jd9xhhmU_bpiJKOU942Wha-OsBiwAXzoS0aWuYMXW_WxihaaV_cmeL1qXuBKkuA8P5H2BegpKei29bmPgekuiTeiAv1-Vl34T4zXouhWLuvPtr2Z0QlDW0dRodYrER3vfg69Y3clltkUF94qEwhNlWzn8jdIf7kAKIeoyEVsn05uHbwCNYUsHoqXfq-mv3jM4md_HffByU6EfFgwCDf7i0u5VCQgEKhBgDPIPJRnEEdeJL_YBM7blI5qGlR8zi63zOxseDReMeeWVpe2kvYTczO4tTYTpjVbUaqVXmSY99id1MINYfgdHDvhCI2ByxSKk96PAYAsZD9Bjo7zte7CIcJB1WDWWkpFlYiyV7XR7xqzp_NZK8082hkEj1JtK39X2BHbsi7dV_uL-JdFq59UFf3ji4KZJ8NpiOZ_rLg2JN5MnrvVB6wdfdOfnAqOL6Q-zgwQk35RB4wil8IM70MKNAPeJcwhGENkkV1Zfovb41ssav5F3BOK3yYEoC5h2LUTTlxoyapUqWB3yYpvQhgF-UMnXAD7qutuFtILR3UPDGZShfEOVpQx6sRUYemucmtGJtyihgw-cSsA9VuSMHni7vDyR1GRbWL8UMuNoIzoIELjPwZ8zmtW8ANK5xwocDHwFcoHEJmMQ0xIHkdIrx3Zj6PvvmXsH9lBaOphaOnXomyYIiYougDyAQDOMeYbDImBNw4vq3iY6LQPMC2j33wkil3zEFxDcVZa965PgoXC7zdszadsa4PECkqyb9ruMKa5Xj9Mx-H7FkHV0000)
|
||||
- **Notes:** SkillSwap Platform is the SoI; Remote Worker is the primary actor.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: SkillSwap Platform (system), Remote Worker (actor), Service Listing (block), Swap Session (block), Mediation Record (block)
|
||||
- Associations: remote_worker→skillswap_platform:registers_on, skillswap_platform→service_listing:generates, remote_worker→service_listing:creates, skillswap_platform→swap_session:manages, remote_worker→mediation_record:initiates_for
|
||||
- Constraints: No money changes hands inside the platform, Identity verification via LinkedIn or similar at signup, Disputes capped to mediator review — no chargeback equivalent
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The platform assumes that users will actively use the 'reputation_score' property to build trust and facilitate matching, rather than relying solely on explicit skill listings.
|
||||
- (0.85) It is assumed that a two-sided marketplace structure (remote_worker -> service_listing) will generate enough supply and demand simultaneously to meet the 'req_002' matching requirement.
|
||||
- (0.75) The model assumes that users are willing to participate in a structured dispute process ('mediation_record') even if they feel their exchange was unfair, as long as it avoids chargebacks.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) If the platform relies on external identity verification (identity_verification), a failure in LinkedIn's API or service availability will prevent new users from signing up. [identity_verification, remote_worker]
|
||||
- (0.85 · medium) The model lacks a mechanism to track and enforce the 'reciprocal' nature of skill exchange, potentially allowing users to list skills without a corresponding need. [remote_worker, service_listing, req_002]
|
||||
- (0.95 · high) The system does not define how the 'reputation_score' is calculated or updated after a swap session, risking users accumulating unearned reputation. [reputation_score, req_001]
|
||||
- (0.80 · medium) The model does not account for the logistical complexity of scheduling and managing multiple sequential 'swap_session' blocks required to complete an end-to-end swap (req_001). [a4, swap_session, req_001]
|
||||
- (0.75 · high) The 'No money changes hands inside the platform' constraint (no_money_exchange) requires a robust, non-monetary dispute resolution process that is not detailed in the model. [no_money_exchange, mediation_record, dispute_mediation_cap]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-003 specifies enforcing a maximum exchange scope of two hours, but the 'scope_hours' property on 'service_listing' is not explicitly constrained to enforce this limit. [req_003, service_listing]
|
||||
- (0.85) The association 'a4', which describes the platform managing a swap session, does not account for the reciprocal nature of the skill exchange (i.e., it only links the system to the session, not the two participating remote workers). [a4, swap_session]
|
||||
- (0.75) The 'remote_worker' block has a property 'desired_skill', but no requirement or association mandates that this desired skill must be linked to an actual offered service listing. [remote_worker, service_listing]
|
||||
|
||||
---
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.30
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TP5BQm9148JlyokQzIp8-i2032Q2UCefzz0xq-CkpaEwnyWI_EyPSG-R1AzVTLLrqrl9oFaIl99J5y_86A11SpfokaIxIpunJDoI7DMZ2MdHfciNZ-3G2uqKur44lXN0sxyKwz3DxafYIppGPeqlwxhGL-Pq7U2AbdPk2zodcDyn45G7yb-KEuELketwOihqf6v6SxDuKdTJRMS9A9np3u52GonAFPQlThtndMJuQCk55Q20UP3FV2UmsKWlcOBMCAdD0bVD7-VOKwRHvCHQb_m1JQVwTwHIMugsVE47)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclist (actor)
|
||||
- Associations: cyclist→cyclist_thing:
|
||||
- Constraints: _none_
|
||||
- Requirements: _none_
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The 'Cyclist Thing' will be perceived as useful by the target user, Cyclists. [cyclist_thing, cyclist]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The model lacks specific functional requirements, making it impossible to define a measurable success criteria for the 'Cyclist Thing' system. [cyclist_thing]
|
||||
- (0.90 · high) The model does not define any specific use case or interaction between the 'Cyclist' actor and the 'Cyclist Thing' system, leaving the core value proposition undefined. [cyclist, a1, cyclist_thing]
|
||||
- (0.85 · medium) The 'Cyclist' actor is too vague and lacks specific needs or behaviors, preventing the definition of targeted features for the system. [cyclist]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association 'a1' links the actor 'cyclist' to the system 'cyclist_thing', but no specific interaction or purpose is defined. [a1, cyclist, cyclist_thing]
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNRRjj647tdLmoSbqGmLLfsWhGmZ3XfYW9fugGAaCV5a1nI2-s5tLrA4583-OVa2_Cb6IufYt4feoy2EBDxpfapCwiNFg0BhLQ9NqlJe4CDEHRhsjdMbA-iiWuULK-feiNeX5zXQRVIr52XyZJA50gzXyy9mCl_e9GLLbcNiQuaDqJpv_ZhymL7hvspso7uZ9wMvoK7_x0cl4DDSF8dgGq5MU19SZlYCpRG4RgCpekB8tItoKgM1AIRi0DDEYVdawGNV_9wGoOiWoFK9u0UgFlsyHikB_tE1z9NL_1eKLpWipooDyu6MrWbDWmahO7VW2-mGPphx6mSUTyw4bxM1XMdSsiLeM4b0-jxPyks80VNJQEupi0maRuP4W8F2TQ1HR2EPNJq2atTOarZsgXQE0nTrBHTYGUoLzRuLa-G5KFY0JBMKt0rp78FlTDHYy1rEJG5JHKPhONVCU31ML7o9tFaoXRhFGVkbCLoN4-GcdXOTNC8ti6-75aCosXnn9-o_cSYHWojPqZY8ZxVl3bT_dNpwNJvyURzBMTjxibjg1HSsDZ5FUydwIWjh6xOfrmHR6LOmHkirdZNjRlHW8hdJylQzPvwh2ZieifrTq5i6KliiQIIF9YNbrqxWaDfGjHxL_2IosU-N1fqkrFm_IUVhYJlf5Hq2gur7fW8zrbZUUBR1eA5ionR3sFzYwSYgkXFYHvIbDGekzDTjuwhk6wmM14ivXdmh93n594mXiL2P688Js4aqtFzRvh-j73SR3Ab5trCR2GcapEQfbVtxWt7lAYSrSauDsEEwQqaBOEVLjOHJ5yQfiGdHnivdzyN5v5PMx39KUCVl3LXyeFd4qKr6mbErgl0epsR_Kt_j2ore_ApMHzw_I7DihFP3Bv_-GOVkRVzQmUwzQ6Rv4ttQbRI-P3sCp1sX3kbqV0RxI6iCF0F7TQWGBCt2gH9SpJbVCIv63Zt3rDQEicbGANijkjaSBn1KKLE4RYa2biLm3RKRnItGlE4Zq7FZnJYswQnBiJpyGdp1VA2FExMulkNhsVPs_robBXxCaQyE8AOTxl__z-2USAjw1kIlE0V_l_z0G00)
|
||||
- **Notes:** EventStream is the SoI; Producer and Consumer are actors; EventData and BackpressureSignal are first-class blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Producer Application (actor), Consumer Application (actor), Event Data (block), Backpressure Signal (block)
|
||||
- Associations: producer_application→eventstream:sends_events_via, consumer_application→eventstream:consumes_from, eventstream→backpressure_signal:emits, producer_application→backpressure_signal:observes
|
||||
- Constraints: Wire-compatible with Kafka producer protocol for migration safety, Single binary, single config file, runs on a single node up to 100k events/sec, Apache 2.0 license
|
||||
- Requirements: REQ-001, REQ-002, REQ-003, REQ-004
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) Adoption hinges on the operational simplicity of a single binary, suggesting that platform engineers prioritize ease of deployment over feature completeness.
|
||||
- (0.85) The target user group is assumed to be technically sophisticated enough to understand and implement first-class backpressure semantics (req-001) rather than relying on existing industry standards.
|
||||
- (0.75) It is assumed that the operational complexity of Kafka's ecosystem (ZooKeeper, Schema Registry) is a primary pain point for platform engineers, making its replacement an immediate necessity.
|
||||
- (0.60) The system must be able to handle both moderate scale (req-003) and maintain wire compatibility with the existing Kafka protocol (req-004), implying complex feature parity requirements.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Attempting to achieve both single node deployment up to 100k events/sec and wire-compatibility with Kafka's distributed protocol may introduce significant performance bottlenecks or complexity. [single_binary_deployment, kafka_wire_compatibility]
|
||||
- (0.85 · high) The model does not specify how the 'Backpressure Signal' (backpressure_signal) will be transmitted or integrated into the existing Kafka producer protocol, creating a critical integration gap. [a4, eventstream, backpressure_signal]
|
||||
- (0.95 · medium) The constraint of running on a single node up to 10M events/day (implied by target user) conflicts with the stated 'single binary' deployment limit of 100k events/sec, limiting potential market scale. [single_binary_deployment, req-003]
|
||||
- (0.70 · medium) The model assumes the 'Producer Application' (producer_application) can reliably observe backpressure signals via a new protocol mechanism without detailing the necessary changes to its existing client libraries. [a4, producer_application, backpressure_signal]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.75) Association a3, which describes the eventstream emitting backpressure signals, is structurally redundant because the block 'backpressure_signal' already has an association to it from 'eventstream' via its properties.
|
||||
- (0.85) The requirement REQ-002 states that consumer-driven flow control must be the default operational mode, but no specific block or association explicitly models how the 'consumer_application' drives this control back to the producer.
|
||||
- (0.80) The requirement REQ-001 states that the system must provide first-class backpressure semantics that the producer can observe in-band, but this observation is only modeled as a general association 'a4' (producer_application -> backpressure_signal) and lacks structural detail on *how* it is observed.
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.85
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLJRRjim37ttLn3jGu1W2RBUq1L1qNROq8TjxOPXhmPjqOaGMVGaELbM1DX7x0lt9QFiNDoqtLw2cAHu3i-XTEK3kb2N9l4pRIjqM4A6-MpYkBRgBHjsS5YSKa57dGe_HSKBRITGeF7KoUG6lOU71E3cIHTLO36A6NQAt3gQdUERyoE9NZl7YtNmZ4xLiPBWUxRX4vO41xTavXHqZWV9geDdET0BS2CwpazUW5ibKwq8gAp24aegCt8-ILho1_SKuAj3wmq6TWU07YeAQTY6O3psIn-elBo4mwFy1C-oXWFMIdCgrEgG-k18-5s0_16X93iNn4ss85BTdBQ1NEKeOD1iKww3LEqALbrgTmjBErOSluGGva9C-4GU3TjbHTA0R5q-A9wyoZ6i94K_AdAQR4wfeJcP5p2-SOv6_spORA7cdQ0WPeRpsHgndLnXmEvKXQFljM0jKrzH7bnTlW1tmse97z1EQfpG5Y-JQ6hMqJt0Linzm8ezfJDQLYo2-YxO1ktE3XJDTKuYUK4k2W65E5evFSTysM2B0MrDkgiPZtEsqWsbSKDXo_-MBVa0c8bf4AO4IXURKnc51P657fNSgDQSqOMiSVpi0Le5lSrYnKX3ODgqJ9NsKg_BnjydyCdE_S7WSdz3Bw3sv9EzwARmaPCNCZpdvFzH_zY9xe7dj1eEdw63LMKq-JHmydIy_vr834r8f79wCWtYSx__HJPBEof9VExtsz2xpuFHw7M_3tz-_OPxnt57FA2u87tcc1bw5LKfQhDttu2BnZcXqFFmI9neK9mUDjCFErX7QwnhOtZXcnQj8O4ZK1FOHsfljXHaBAqNEanZcNOHlDlxUDtxOotBnN9fKHwwM1xNhAuCeuhlhtnwch9HxHK4KQwv6X2lnZ0HqLhfaYlvaMV-Bm00)
|
||||
- **Notes:** The Pet Translator is the SoI; pet owner is the primary actor; vocalization and body language are first-class data blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Pet Vocalization (block), Pet Body Language (block)
|
||||
- Associations: pet_owner→pet_translator:uses, pet_translator→pet_vocalization:processes, pet_translator→pet_body_language:processes
|
||||
- Constraints: On-device inference for privacy, Honest about the difference between 'emotion classification' and 'translation'
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that the 'interpretation output' will be perceived by users as a genuine, useful translation despite being based on scientific emotion classification. [req_001, honest_disclaimer]
|
||||
- (0.90) The product assumes that the 'interpretation output' will be sufficiently believable and useful to fulfill the desired outcome of bonding with the pet. [req_001, req_002]
|
||||
- (0.85) The system assumes that users will accept a tool that requires continuous input streams of both audio and video data for interpretation. [pet_translator, audio_input_stream, video_input_feed]
|
||||
- (0.80) The product assumes that the technical feasibility of inferring pet emotional state from combined audio and video inputs is achievable with sufficient accuracy. [pet_translator, a2, a3]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The system's core value proposition, 'translation,' risks failing to meet user expectations if the output is merely an emotion classification, potentially leading to negative reviews and abandonment. [pet_translator, honest_disclaimer, req_001]
|
||||
- (0.90 · high) Relying on the combination of 'audio_data' and 'video_data' for interpretation requires robust, real-time multimodal fusion that may be computationally prohibitive for achieving 'on_device_inference' performance. [pet_translator, audio_input_stream, video_input_feed, on_device_inference]
|
||||
- (0.85 · medium) The 'interpretation_output' must be sufficiently actionable to fulfill the requirement of helping owners bond with their pet, otherwise the product will fail to deliver its desired outcome. [pet_translator, interpretation_output, req_002]
|
||||
- (0.75 · medium) The system assumes the availability of high-quality 'pose_keypoints' and 'frequency_spectrum' data streams, which may be difficult to capture consistently across varied household environments. [pet_body_language, pose_keypoints, pet_vocalization, frequency_spectrum]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The association `a2` (from `pet_translator` to `pet_vocalization`) and `a3` (from `pet_translator` to `pet_body_language`) imply that the system processes raw data blocks, but the `pet_translator` block's properties list only includes high-level streams (`audio_input_stream`, `video_input_feed`) which contradicts the detailed processing of internal data structures like `frequency_spectrum` and `pose_keypoints`. [pet_translator, a2, a3, pet_vocalization, pet_body_language]
|
||||
- (0.85) The requirement `req_001` states the system provides an 'interpretation', but no block or association is defined to represent this final, interpreted output beyond the generic property `interpretation_output` on the `pet_translator` block. [req_001, pet_translator]
|
||||
- (0.75) The association `a2` and `a3` define the input flow, but there is no explicit block or mechanism to combine the processed data from both `pet_vocalization` and `pet_body_language` before feeding it into the core logic of `pet_translator`. [a2, a3, pet_translator]
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.95
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLR1Rjms4BthAmRE9LbuZUqw2P95OAHnqoG7fqqQ84U2aaPQTYcI9KUxsGO6-X7zmdv9XvIqLjQh81VRUEJCUvnvC_BBGD9Js-WiR9Hnqii6SbbiQczRKrvRRJqyg9vWXSlHZR2MfTqfKqCbTS3HIQ5b2F0r0tXrBqjPoMeHJwmlqVTe_am-VxPaz6VlxQu7d-AJyh9ay5ThwBri4CxUejuYgKAUPRSZFcC99-WMU5aydg2xpTQgHC36qHuQR7Bq8SiwyMSVMeK4Rsth0_okfJ7epq06-2lYOXrn4VQ1i84NBxe_hgxWmR9uB9_cIOppBAiWB4KerbYs6c45WJnNZ4yDrfQK96MDuF9J6_WKJTjyTMXAldCkYm8TXvzx_3FbkMN5lRftV6MhobPgkERgU9MtP3k1wd0aYhiZ5YaB_iqQexP2QcQHNaII4AbpQpLA4yziuwJP2s_7YW_Khqsj3684uKOQMVU5mJjSD1r-Hpe4N_Uyy0VwhIemHHx4X0xai5pROjDhbKu9sZiSwbDRMsiKGzIvRKbRkn6rbsvzom6a6XH_MuCYjCvPJwFd7PvnWsmaVi8xirNKwMWwIAW47Qb88o9Ywd4NMS4zJ18nZ1XE5EyNH0UVhT_0gvWxqV9L9tQCYSHtn5ksFfc4a_7ZsQACgI0EvlhcRGFDHvJbVEVZAzwaQi6XyDAKy5jNCNXuemflWwq8tiJYFMBp4DOTOTBcEL7A8qOdh92TnK-J32QPTrLj0t5GSAmjzI6s0KfLLUZHiDK0lpYj2aLwpsHnHfmDl3uIsn0WXWpJHEyjIFPrmu8AIFMApTmfuiaaQDb9x6DJATzqaYiU0LeZCC8ABnTGods0XtchT0daMVQZVdeKJwKM2HAExMpBKpAoYJ6RpwzEhOXLb1Io8uTDNkuQ72UJjqU-pqvSYh7t1cTrQ2064Oin5NdikHNKo1iknas5J3nurVUG2OSr9xZOsMddN5nCl5WwjWNd8FkTo1Fl7SNTSy-Fa9qCEchJDm4d3V8ZJ9b6hd49NjLhuY6UpJuYntXiq52OpJhezOVvOl7JR0R__VClV6B3zgcQE5XePCwp77sS74wMPqqh1lVSF-BD0xm8-E8ocfzKtTgsso_psEjkJWAmQDWXRYv6bCi9odxFf2GmJZACMc53sgC7KSvg5J_Hcjlb54azfhcSe1c-XWXI4th3FT_Yq8Uq5tWxhFbzIK8SvjpI6lZpJrpUYumBsvKtUya_-3-P_m40)
|
||||
- **Notes:** The Quiet Hours Planner is the SoI; individual contributor and engineering manager are actors; calendar service handles external APIs.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours Planner (system), Individual Contributor (actor), Engineering Manager (actor), Calendar Service (block), Meeting Invite (block), Deep Work Block (block)
|
||||
- Associations: individual_contributor→quiet_hours_system:uses, meeting_invite→quiet_hours_system:is_processed_by, quiet_hours_system→calendar_service:integrates_with, quiet_hours_system→deep_work_block:generates, quiet_hours_system→individual_contributor:proposes_schedule_for
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that users will prioritize the protection of deep work blocks (REQ-002) over maintaining existing social relationships or meeting commitments. [req-002, individual_contributor]
|
||||
- (0.90) The initial hypothesis assumes that proposing alternative times (REQ-003) is sufficient to resolve scheduling conflicts without requiring the user to expend any social effort or compromise. [req-003, individual_contributor]
|
||||
- (0.85) The system assumes that integrating with existing calendar services (a3) will provide the necessary data granularity to identify and protect contiguous deep work blocks of sufficient duration. [a3, calendar_service, deep_work_block]
|
||||
- (0.80) The system assumes that the target user base (individual contributors and engineering managers) will adopt a tool that actively intervenes in their existing meeting culture, even if it requires negotiation. [individual_contributor, engineering_manager, quiet_hours_system]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's ability to negotiate alternative meeting times (req-003) is constrained by the 'initial_trust_period' constraint, which prevents automatic blocking or proactive scheduling changes. [quiet_hours_system, req-003, initial_trust_period]
|
||||
- (0.80 · medium) The model does not specify how the 'negotiation_status' property on the quiet_hours_system tracks or resolves conflicts when multiple conflicting meeting invites are processed. [quiet_hours_system, meeting_invite]
|
||||
- (0.95 · high) The 'calendar_service' block must handle the complexity of reconciling time zone differences across multiple attendees and protected blocks, which is not detailed in its properties or associations. [calendar_service, timezone_respect]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The constraint 'Read-write Google Calendar and Outlook (Microsoft Graph) integration' applies to both the `quiet_hours_system` and `calendar_service`, but only `calendar_service` is a block capable of performing this action. [readwrite_integration, quiet_hours_system, calendar_service]
|
||||
- (0.80) The association `a5` states that the `quiet_hours_system` 'proposes_schedule_for' an `individual_contributor`, but no requirement or constraint explicitly defines how this proposal is communicated back to the user or integrated with their calendar. [a5, quiet_hours_system, individual_contributor]
|
||||
- (0.95) The requirement REQ-003, which mandates that the system must negotiate alternative meeting times, is satisfied by both `quiet_hours_system` and `meeting_invite`, but neither block contains the necessary properties or associations to model the negotiation process itself. [req-003, quiet_hours_system, meeting_invite]
|
||||
- (0.85) The `engineering_manager` actor is defined but has no associations to any other element, nor does it satisfy any requirement, suggesting it may be an unused element. [engineering_manager]
|
||||
|
||||
---
|
||||
400
phase-0/reports/2026-04-28-1942-run.md
Normal file
400
phase-0/reports/2026-04-28-1942-run.md
Normal file
@@ -0,0 +1,400 @@
|
||||
# Phase 0 corpus run · 2026-04-28T17:42:31.750Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 8 · **OK:** 8 · **Failed:** 0
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 5/3/3/4 | 3/4/3 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 02-habit-coach | medium | 6/5/3/3 | 4/4/3 | 0.90 | 5 | 4 | 5 | 5 | 4 | 5 | 4 | **4.57** | **4** |
|
||||
| 03-redline-ai | medium | 5/4/4/3 | 4/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 04-skillswap | medium | 6/5/3/2 | 4/5/3 | 0.90 | 5 | 4 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 05-cyclist-thing | failure-prone | 2/1/1/1 | 1/3/0 | 0.30 | 3 | 5 | 4 | 3 | 4 | 5 | 4 | **4.00** | **3** |
|
||||
| 06-eventstream | hard | 5/0/0/0 | 3/4/1 | 0.92 | 3 | 2 | 4 | 2 | 4 | 5 | 4 | **3.43** | **2** |
|
||||
| 07-pet-translator | failure-prone | 5/4/2/3 | 4/4/4 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 08-quiet-hours | medium | 5/4/3/2 | 3/4/2 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
|
||||
**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed
|
||||
|
||||
**Passing:** 7 / 8
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRXCn47tVhnYrBn2HAAGj0XHLb0h40zSAdgrPUtPZnMillYH4g18VmHVo9OntatRR9dr8f8p7vnoVENRUX8W-fiOKOQbjYnuRA54kQ--ILPVEE0_7rHbLD1jqX0KgjzQsXWfDeC6AD1W2_2u0tZv0KHLMqxpYl2A_hPOlyTNB6LSll7VhRV45dQaJnSNtpiRFs10SVI2pegWb7XNN0pxh8XsWczA9F3r0Trqij2AWfeqRQAWfoOUYwCKVNNWTeek6ZW03uEuNpETX4o8rv-TmF9Ed-ABicBKDKSSKjRDeH9abeD_0QoYTCuIMMunR2pPwIK5e6ycZpDs3bkiTzm_BMckFAc4a9ezWqDS4RIgDbf2iNh5K7JM5JbqQzZD7KcGZAsK6vrbe5YXTye7MpYz5mrrUes7g43qFa9UL3cHhhAdXhO9Fk-HLcx8djxekRp3WqnQZuz-7FP-NnidbbhtrhZJKY41no1fTgwL0JtXRlM7xI38cJt39usTH7PFfQqBsjNiimV0ymv1WIPlSAwcDOG_5bwhIak13rj_XkwuW5hbm3nnNg0sMskYOvqfipkyW5wII8VKCrNZXv3L3xx0_fH314ayAN8fjYe1iLDCQuh06Pxgii0DSpgCZjJEf4zBmRi6v5jov8d7JFv_pSVWOo17gXDsSvOf2oql4MM5hvAPF1JoVp09yFPjcfjPJ8ByYg3YBlPzzhsYvuJ3q-tTNNo-0Ep7Z_llpTp3nxH4vm2ks_iurWDqrwAWgyYsA7SWUic9_ZCVZyy7rUmqfy0N28E92Mtu1Qi5lGBGSqzkcl6L_tjbYIO4H7Y6xdx4ScwJuhedAkwOuD9d9vAvER5kJDqLN35r-fEskGuyq5ePgiWgyhXUHepaQNT7Ff7rtUSDer9VUVHjFfyz7esvWtnSSkkxXwgALZLZbnm_pipJkdoNOMGfRIu5-jUmSQwV907Ds49EZm8P5hcyZxwg7cGyvYdSZ7uRm9mVWcvmL_e1_FEb3hDCjreKnmymgZDY3icSPSgtpsJlb-oCCJ_fBq9r6yu3ewQJW6VIJADxm5_z__WS0)
|
||||
- **Notes:** Aristotle is the SoI; students are actors; course material and lecture content are key blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Undergraduates at large public universities (actor), Coursework Material (block), Lecture Content (block), Office Hours (block)
|
||||
- Associations: undergraduate_student→aristotle:uses_as_thinking_partner, aristotle→coursework_material:processes, undergraduate_student→lecture_content:studies_from
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002, REQ-003, REQ-004
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that students will adopt a tool that explicitly refuses to solve their homework, because the market is saturated with answer-givers. [aristotle]
|
||||
- (0.85) The system assumes that students will use a low-stakes thinking partner during the long tail between lectures and office hours. [undergraduate_student, aristotle]
|
||||
- (0.75) The system assumes that students will use coursework material as the primary input for the thinking partner. [undergraduate_student, coursework_material]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The combination of FERPA tenancy and the need to process coursework material (a2) creates a high-risk dependency on institutional data pipelines that may lack necessary APIs or bandwidth. [aristotle, coursework_material, a2]
|
||||
- (0.80 · high) Maintaining a low latency (latency_p50) while processing complex, unstructured coursework material (a2) and adhering to strict FERPA tenancy requirements may be technically infeasible. [aristotle, latency_p50, coursework_material, a2]
|
||||
- (0.70 · medium) The system's core value proposition (low-stakes thinking partner) is highly susceptible to market saturation, potentially undermining the initial hypothesis if competitors adopt similar refusal mechanisms. [undergraduate_student, aristotle]
|
||||
- (0.60 · medium) The model does not specify how the system handles or structures 'disengagement risk' (undergraduate_student) beyond simply processing coursework material, risking a lack of actionable guidance for students. [undergraduate_student, a2]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association `a2` suggests that `aristotle` processes `coursework_material`, but the system's purpose is to help students re-engage with material, implying a flow from student input/study materials into the system, which is not captured by this unidirectional 'processes' link. [a2, aristotle, coursework_material]
|
||||
- (0.90) The `undergraduate_student` block has a property `disengagement_risk`, but no requirement or association utilizes this specific property, making it potentially unused. [undergraduate_student]
|
||||
- (0.80) The `office_hours` block is defined but has no associations linking it to the system or student, suggesting it may be an unused element. [office_hours]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (student) and system blocks (coursework, lectures), but missed explicit mention of 'thinking partner' as a key concept.
|
||||
- **model.accuracy:** 3/5 — The associations are generally correct, but using 'processes' for coursework is vague; a more directional flow (e.g., analyzes) would be clearer.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids excessive filler blocks, keeping the scope tight to the core problem domain.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (FERPA, latency, no solutions) are correctly captured as both associations/requirements and applied to the SoI.
|
||||
- **asm.detection:** 4/5 — The assumptions are specific (e.g., low-stakes thinking partner use), tied to elements, and go beyond generic market platitudes.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (FERPA/latency), and directly linked to the system's core technical challenges.
|
||||
- **voice.character:** 4/5 — The findings are structured, skeptical, and concrete, though some prose could be slightly more critical of the underlying assumptions.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run; the risk detection was excellent, identifying complex technical dependencies. Minor improvements in model association clarity would elevate it further.
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNTRjDA4BxtKupAJOb8AMr1FHMgA04EYByS2jUhyNfihxBUzTbT9rGe4W_14vudERDhXpeDwKtLpD_tpSmtwnS-e0jTepE_KAP5XmtaA1ULivqfPbPR1m_AfrJIoIZ2rrZObJ8LbAWzZJnIe_Vm8mDukLEbAB4yZXxh2dA3DJ_7lyvFs7hbd5qDncVqj3WjsFZ6cl096uA3jwIN59J4WsmzmZCsq1wuOpgLPtlWrbcj2W9gsd033JKvEPzbFVc3jvYh03EBiZu0z537tqB6t_3ykR_nWPhBItXm8i_mMPxGcGoP80hqTMxH5N01kRMQqB1JraeNOgbeDJAlDsZpb9g0-YgCWJ9Onn2nDE-deY2m8X7xQxa6cQxvSJvzShQ-BJEB8AbAWhijiWcubex8m5V5qto50LEiJqOHsCZa0t9QhgrS3DYTZx0D1QUaPs3F_vYA7LerF1BMJkZyXj1AXLePaLp-4NsNH0KLm-yHqxd6K5hNmDNy7HogpzUF_xQsqlImdy7W5TYgEsoK2LGv3CgQM-YTUSwiMV8sKnXyi5K_av5HQ5lTgHnig-IuFvu52KSoIgUuTQn_Q-HZvmEih5i0Qcig7zN4hS1uocWAw5k3DzWeVGCVb5ayYfe5Hzoe90WrDObXmmN5faCISPZ2UkR9r1aTsPN8RYySqj9YBkGqw6VA6S5PhPbvImvcd-VpApZyTlsmP_Hg_cNw7WvVV-i77Yk8rgaboXiMUTDgXOQPxKV-PC5RgL13IRm5HnvuOx1GXOV_VlxY_e1NmZUNJi77Xg7jSiqZveSZ3Z816olwCk9tcJzWPiDHJAUNMuTu4NTQgYhcPRtaTqFYpVaaR65DjYMjtU2IzJe4BnLcO-TaEjqzbGjedPNaFHEu6xoZjmkeo51ShS_shFhew2uZR7aPv4Mmz-LijRMLj6-rzoBTbpJiOYl-3skyjtwcYKTHW5DL7VZK9fClz6-d73LSsayclUdrFzFZuoUJIHBKrvgWVtPxgPSebLRy0l2D0R-9GHc25T62JsmyURw9i8hdEF2EyazIEHg1d8n0jf8tLzL_LVYOb4dwfVxg-ZkE8cHXXt75qpsqVTUsreNnEmPnaK3VbG_nQyeFI2ypY3BZPybgVvJnjFgPPI_u3t-h_mS0)
|
||||
- **Post-validate:** stripped 1 refs, dropped 0 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (actor), Screen Time Data (block), Platform API (iOS/Google) (block), Conversation Log (block)
|
||||
- Associations: parent→habit_coach:configures, child→habit_coach:views_data_on, platform_api→habit_coach:feeds_data_via, habit_coach→screen_time_data:processes, habit_coach→conversation_log:generates
|
||||
- Constraints: Must work alongside iOS Screen Time and Google Family Link, not replace them, Child data must be parent-controlled per COPPA (US) and GDPR-K (EU), No social features for kids — no leaderboards, no public sharing
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that parents are willing to adopt a tool that shifts the dynamic from enforcement to shared, data-grounded conversation. [req_001]
|
||||
- (0.80) The model assumes that the 'Conversation Log' (a5) will be sufficient to facilitate a routine weekly conversation. [req_001, conversation_log, a5]
|
||||
- (0.90) The system assumes that integrating with existing OS tools (platform_api) will provide sufficient data to ground the conversation without requiring deep, proprietary access. [req_003, platform_api, must_integrate_with_os]
|
||||
- (0.85) The model assumes that the child will actively view and engage with data presented in the 'child_view' to make the conversation collaborative. [a2, habit_coach, child]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on the `platform_api` to feed data, but the model does not specify how it will handle discrepancies or missing usage metrics when integrating with two distinct OS systems (iOS and Google Family Link). [platform_api, a3, screen_time_data]
|
||||
- (0.85 · medium) The `parent` actor is responsible for configuring the system, but the model lacks a mechanism to enforce or track if the parent actually uses the data (`screen_time_data`) to initiate the desired weekly conversation. [parent, a1, req_002]
|
||||
- (0.75 · medium) The `child` actor views data via the `child_view`, but the model does not define what specific metrics or visualizations are presented to the child, risking the tool becoming merely another source of parental judgment. [child, a2, child_view]
|
||||
- (0.95 · high) The `conversation_log` is generated by the `habit_coach`, but there is no defined process or trigger for when this log is updated, making it unclear how the 'routine weekly conversation' (req_001) will be sustained. [conversation_log, a5, req_001]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association `a2` (child → habit_coach: views_data_on) is structurally inconsistent with the constraint `no_social_features`, as viewing data implies a form of visibility that could be misinterpreted as public sharing or excessive child autonomy. [a2, child, habit_coach, no_social_features]
|
||||
- (0.75) The requirement `req_003` (system must support integration with existing OS screen time tools) is satisfied by the association `a3` (platform_api → habit_coach: feeds_data_via), but this relationship does not explicitly account for the necessary bidirectional data flow required to 'work alongside' external systems. [req_003, a3, platform_api]
|
||||
- (0.70) The block `parent` has a property `target_age_group`, but no requirement or association utilizes this specific piece of information, suggesting it may be unused. [parent, target_age_group]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 5/5 — The model successfully identified all core actors (Parent, Child) and necessary systems/data sources (Platform API, Screen Time Data).
|
||||
- **model.accuracy:** 4/5 — Most associations are correct, but 'views_data_on' is slightly weak; a more precise association kind would improve accuracy.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and avoids adding unnecessary or fabricated blocks to support the core narrative.
|
||||
- **constraint.capture:** 5/5 — All three constraints are explicitly captured in both the dedicated constraint block and applied correctly to relevant system components.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., data sufficiency, adoption shift) and tied directly to functional requirements rather than being vague platitudes.
|
||||
- **risk.detection:** 5/5 — The risks identified are high-severity, domain-specific failure modes related to integration complexity and behavioral change.
|
||||
- **voice.character:** 4/5 — The findings are highly structured and professional, maintaining a skeptical tone while remaining concrete and actionable.
|
||||
- **Average:** 4.57 · **Min:** 4
|
||||
|
||||
> This is an excellent run; the model structure is sound and the critical thinking in the findings section elevates it significantly.
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/fLRRRXit47tVhnWuRuAakd8I989Wn7MQra3IDh61FX8KTtP5cvSjoPMY1WRw4Vt2Vac7ors9SYmZG5uaxN0kXpFdpEgD3zo5Lgl2tqdJSCSrBBcugvrjJNbfbNNmh7g15Suo3x_YfTr8KqF5bSViH2ZkFNmf07xwAajPyUeqdbXNekkjorVyzQiPMI-SivlU-19Vb6Sb6TzP4txZ6k7aLrHh35Bma-8-gsTim2FbJl5CF3zIxhvOoH81THEse54lqVcYIE1FFc6ff46uk3e1xi6bHyObB1P-wmFgytDuDXFF-SjbLrkQWBNZGLh36iL3PPs6EVZWg4LqJdbRZqp9fUDkoxWGI7NciBHM8JVamTjI1XOSbuhvO1slCJk_7v2znvehk46kEsGgFh90ZuIC2ueZO16GepYAHzFgBpKQTEGcg3SUrVYsDT8wHlcia3pW_Jxz9NNRKHvuQqMhqOIkYXYiYyLIMN7NroXx7nQs3HxSblp9lZVjytTzWFUf3odvGMiUb11MDzniMNBoUH6YhIcvA-NVgUqArwY6Aokx6MiiPQl7ArcliXiEexrkwnfzZ3mOiT-P7o3f3x1aOiLDVN3VsvOYAc9c5tU8CRH-0DMWASa-ZjDlADFOuIqA-X7X3VW-IE6ijrM0FucrSBKdLeTJKoasi0tPHeOLsqJ7nOAwxYD_JEWXtwmG4bkxCB0DnY28GT90eECFrwdCC4e2kfREclXxkc_QMnuu_E7acelj3_5R4l6GTDN64QGv6iiYSiy4rR8QNKTc4rV48zXsH1kuqY4qD4C7dIDqoIP3Cj2sHEKpJ1THC70J1GEVyAzMEjmH5ZytrWM-LAJPlQuUmt5Xal8WUTWA8eGjz4FEwD01N6AMUvhhHZTASYE8M3qlEo2ai5ad3oOb4tkdRvYMZcUn4TU_Ny8CRgB6hguWIr8vgw7aMx2cmr9ass0oES_NrXoYV9X7jvO2Y-maEkxqFOV6sRYQq7_bSoZJEMMlq27XE70TJIQFQMaEQGa5yZvAuUdq0MRUq5K9Th354-pwZh3iubdG4VOyLUVOd9-AYUEYZG2E5bGW0en66OtzQ9HCFt-Sd9x-E1h1V__y2--ays7IHChrMJqDVxSAT_T9Uo9n1Kdnn28jJKlTdsPPPttMoBnVqZk2TDcz8rAymxN43Rq5CnruIaOavMNK2c_3oiPTjuVJaT8B6grOvSNEic8vpObmw_WYZdgGlbi5NO_wPKf5-dV0n3SeufRjgGY9YZBUYTgOcbcyeG_wB_C_)
|
||||
- **Notes:** Redline AI is the SoI; Legal Team is the primary actor; Contract and Clause Library are key blocks.
|
||||
- **Post-validate:** stripped 1 refs, dropped 0 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Clause Library (block), Redline Suggestion (block)
|
||||
- Associations: legal_team→redline_ai:uses_service, redline_ai→contract:processes, redline_ai→clause_library:references, redline_ai→redline_suggestion:generates
|
||||
- Constraints: Microsoft Word Integration, Data Privacy/Privilege Boundary, Audit Trail Requirement, Compliance Standard
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that the General Counsel (GC) will find AI suggestions trustworthy enough to review, but not so perfect as to eliminate their final judgment. [req_002, legal_team]
|
||||
- (0.90) The product assumes that the legal team will adopt a tool that automates mechanical redlining tasks, even if they are currently paid for or comfortable with manual processes. [legal_team, redline_ai]
|
||||
- (0.85) The system assumes that the value proposition of 'minutes' for first-pass redlines is sufficient to overcome the inertia and cost associated with existing manual legal workflows. [req_001, legal_team]
|
||||
- (0.75) The product assumes that the core problem is the time spent on mechanical redlining, rather than other high-value pain points like contract negotiation strategy or risk identification. [redline_ai]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The system may fail to maintain the data privacy boundary if the 'processes' association (a2) allows raw contract content access without explicit, secure handling for cross-customer isolation. [a2, no_cross_customer_training, contract]
|
||||
- (0.90 · high) Achieving the 'minutes' latency target (req_001) while integrating with Microsoft Word and performing complex redlining logic may be technically infeasible due to platform overhead. [must_integrate_with_word, redline_ai, req_001]
|
||||
- (0.85 · medium) The 'Audit Trail Requirement' (exportable_audit_trail) may become a significant execution bottleneck if the system must track every change suggestion and its justification across all users. [exportable_audit_trail, redline_ai, a4]
|
||||
- (0.75 · medium) The system's ability to provide consistent clause application (req_003) is dependent on the quality and structure of the 'Clause Library', which may fail if it contains ambiguous or conflicting company clauses. [clause_library, a3, req_003]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The requirement that the General Counsel must review AI suggestions (REQ-002) is satisfied by both the legal team and redline_ai, but no specific association or block property dictates this human oversight loop. [req_002, legal_team, redline_ai]
|
||||
- (0.80) The constraint 'Data Privacy/Privilege Boundary' applies to both redline_ai and contract, but the model lacks a mechanism or association defining how data is processed by redline_ai while maintaining this boundary. [no_cross_customer_training, redline_ai, contract]
|
||||
- (0.75) The requirement 'First-pass redlines on standard contracts must complete in minutes' (REQ-001) is satisfied by the redline_ai system, but no associated block property or association defines the speed or efficiency of this process. [req_001, redline_ai]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It captured the core actors (Legal Team) and blocks (Contract, Clause Library), missing only a mention of outside counsel.
|
||||
- **model.accuracy:** 4/5 — The associations are generally correct; using 'composition' for redline suggestions is appropriate, though some labels could be more precise.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and only includes necessary components to address the core problem space.
|
||||
- **constraint.capture:** 5/5 — All four explicit constraints were correctly surfaced as dedicated constraint blocks with appropriate application scope.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., overcoming inertia) and tied to the core value proposition rather than generic platitudes.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific (data privacy), and directly linked to technical constraints.
|
||||
- **voice.character:** 4/5 — The findings are structured, skeptical, and concrete, focusing on failure modes rather than just listing missing items.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run; the detection of specific data privacy risks and technical feasibility issues elevates it significantly.
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLRRZjis47tNLn2RbzPO6zlD1QaRB9AcBHeWITjiWJuA53MIMVEYafGT8J3GZ-WNzajwICfU-R900SEMPpXp3izSxDS-S1TwhGg_bgRZZcjMSR5kdUrD_TOgwzYJvZarT3ivuLUyjbjfMjPmvMdY4OfxpxuKZ7r_bgLkU7CJFTRLv4PhzP9_-_8MrZVEsUreV47FwwSrZ3zP4pvoJUpgPr8R2bBmgs8tmJCsq2DmD_HKF7i4RbUiP4sCT1S6fabNv7nHPF9NzskfrFsMT-nNnKDZdRvYt3CVpHxciXlDxDKhF_X0-kwEFRaLp_YBAd6XpsB5JKlbIlfWtS2Mp0S7kU3hFRcoSxQHYlo3OxS7_qGwNkaFwzRa4gvBbdARB83a0Zc1694qXp7mIb6PwKsXagNa6ouLhwII8HAfh5N4pGDWkYtxfMb6k1nbutU0LSgAzGYMFM7eQ8f2dpjoGNeg5Mr8mKMcrrygQgKnvAwb2UGqrP87kiw7TmWBKbCfhDOoQ381OOX26IR4OWNUGq7WP6PHUvKD9zGS_TLBHtLMuKW4OJigQ_B2oIv8QuwvZuN258JUxxaZmbnh7iIAwcjmx1I5-CID8AMetatBrVM8ZNdlXNKqLcnlB7qqdfBbwGQ76qVw6qbRoD5VLE93fCBUEWab9KyGYLqfHjF5GjaDEGN5CeK9Md2z3uTg7HfmZ_RHCcqD3GTr4g2nPRAMUoimXJM8vgZm2FpxYjXXE3HGudnYHuZOVQW_usP8oR5q1l3WngEReTFYuRBlQhH3x5NGbmq6FleJ1Jbwogdd4eqtQ84yDInFZSzaKjnmoCCsKEwzD6kgtnc6xU2bngZWazn62h0DCBMcxoRqVf0UnSNe2jvrLBC0jL9p8pwNCN6jyx4o-nxXUSZ5KI05S7HPgeS8QhYG-Jl8oQ2-QodkQBXZAQJNJ7g6mK82P8XaY-EbCP_VNLfVI-HjCKyGgBJc96Osdn_jWYN3rjf0ErzSI9KGflEvH8iQtkBqUTgp-K1kwpt5rsF9Jnf-6HUfB-4kaoUsy7cBBXQFt1krKnBnmHQNkof67XC_2Zelx_-1AXIrP6hcPBiAgFzizYdlhRW1_MoMJJ_-Dh-v-MOsO__-_K-QgFmRarkOJ5psB6njR9HuIJGtEcM_fHZFB58nSBgU1pkFc5_jzrdUYFZSguJ7oePLN4jeFnH21edLC7pDjZ90M0PDsPPepMp3_hJIGEl5XEhjXEhnyFk-wwmBBC1pg0cI66lcWllLA1YhA22_EPdPHG7UMR7YDTxm9-K_)
|
||||
- **Notes:** SkillSwap Platform is the SoI; Remote Worker is the primary actor; Skill Offer and Swap Listing are key blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: SkillSwap Platform (system), Remote Worker (actor), Skill Offer (block), Swap Listing (block), Reputation Score (block), Match Criteria (block)
|
||||
- Associations: remote_worker→skillswap_platform:registers_on, remote_worker→skill_offer:provides, skillswap_platform→swap_listing:manages, remote_worker→reputation_score:possesses, skillswap_platform→match_criteria:uses_for_matching
|
||||
- Constraints: No money exchange, Identity verification, Disputes capped to mediator review
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that users will actively engage in the platform enough to complete a full, positive skill swap cycle (matched, scoped, exchanged, rated) within one week of joining. [req_001, skillswap_platform]
|
||||
- (0.85) The platform assumes that reputation scores derived from external sources (like LinkedIn) will be trustworthy and portable enough to facilitate high-trust matching. [match_criteria, reputation_score]
|
||||
- (0.90) The system assumes that the scope of skill exchange can be reliably bounded (e.g., max 2-hour exchanges) to prevent open-ended, complex retainers. [swap_listing, skill_offer]
|
||||
- (0.75) The platform assumes that the core value proposition—non-cash skill exchange—is sufficient to overcome the inertia of existing, established cash marketplaces. [req_002, no_money_exchange]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.85 · medium) The system lacks a mechanism to manage the reciprocal nature of skill exchange, potentially leading users to only list skills they want rather than what they can offer. [skill_offer, remote_worker]
|
||||
- (0.90 · high) The model does not define how the 'scope' of an exchange is agreed upon or bounded, risking open-ended commitments that violate the initial hypothesis. [swap_listing, skill_offer]
|
||||
- (0.90 · high) The 'match_criteria' block only lists `trust_level` and `skill_overlap_score`, but fails to model the necessary reciprocal matching logic required for a successful skill swap. [match_criteria, remote_worker]
|
||||
- (0.80 · medium) The model does not account for the decay or maintenance of reputation over time, which is critical for maintaining trust in a non-cash exchange system. [reputation_score, skillswap_platform]
|
||||
- (0.75 · medium) The `remote_worker` actor is only linked to `identity_verification`, but the model does not specify how portable reputation (from LinkedIn or similar) is actually integrated into the `reputation_score` block. [remote_worker, identity_verification, reputation_score]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The `remote_worker` block is associated with a `composition` association (`a2`) to `skill_offer`, but the composition relationship implies that every remote worker must provide at least one skill offer, which contradicts the nature of an actor who may only be seeking skills initially. [remote_worker, a2, skill_offer]
|
||||
- (0.75) The `swap_listing` block's property `required_skill` is not semantically linked to the `remote_worker` actor, making it unclear which worker's needs or skills are being represented. [swap_listing, remote_worker]
|
||||
- (0.70) The `reputation_score` block is composed to the `remote_worker` actor (`a4`), but its properties (`rating`, `review_count`) are typically accumulated over time and should be modeled as a property of the system or an association, not solely owned by the worker. [remote_worker, a4, reputation_score]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 5/5 — The model successfully identified the core actors (Remote Worker) and necessary concepts (Skill Offer, Swap Listing, Reputation Score).
|
||||
- **model.accuracy:** 4/5 — Most associations are correct, but modeling reputation as a composition to the worker is inaccurate.
|
||||
- **model.parsimony:** 4/5 — The model includes necessary blocks (Match Criteria) without adding excessive or fabricated filler entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints are captured accurately and applied to the correct system components.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., bounded scope, trust portability) and tied directly to core hypotheses/constraints.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive failure modes (reciprocity, scope definition, matching logic).
|
||||
- **voice.character:** 4/5 — Findings are concrete and technical, though some risk descriptions could be slightly more concise.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run; the detection of failure modes related to reciprocity and scope definition was particularly insightful.
|
||||
|
||||
---
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.30
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TLBDIWCn4BxdAORsjb1fQoigPQaMnOk2Mi-IJMPtW_cfoMnB4S67y0bz4cTPAwNY9OSlttm_mymJoKYDioAz6B-IKJeef7gfOcYyNWGR8lJBAPOutcEaMkgmCRw2Kjg4Up_AofJWJG1S_b7HfIo7xK-86kCFMfpAizCnenSnXiqFU89JVQmPl0wUxgH3wDsWNICP9NlYVS_F1y9_x8Puh2R_sBsBscW4T2lQWaDNO4n2TE5xYwso9X4iQsxO0vb0TSWpjGZCPccR25sUGt-i9lAaE9XC-qDCbuf2pFDVWoUliJGUDJogiCAEFMAY2ZvHbCOJQxUjbZL2PmQffO9hMB50s0aKM8R8SD1essnAUbZ5e13r4RSKkn23GNxGuXoQX5pQOeLUGpHLJUoOPG_usfY83ZsbBEkWg_l1S3ZAClZw-8JRDWERhDiLDcIiuILI0EAmV4qLKYiTZmJhTkfYpWyVsJS0)
|
||||
- **Notes:** The seed is too vague to generate a detailed model. The system of interest (SoI) is named 'Cyclist Thing' as per the input title, and the primary interaction is between the Cyclist actor and the SoI.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclists (actor)
|
||||
- Associations: cyclist→cyclist_thing:uses
|
||||
- Constraints: Undefined Scope
|
||||
- Requirements: REQ-001
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The product must be useful enough that cyclists will adopt and use it despite its undefined scope. [req_001, c1]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The current scope is too vague, making it impossible to define specific features or measurable success criteria for 'Cyclist Thing'. [c1, cyclist_thing]
|
||||
- (0.90 · high) The model lacks any defined interaction mechanism (e.g., data input/output, physical integration) between the 'cyclist' actor and the 'cyclist_thing' system. [a1, cyclist]
|
||||
- (0.85 · medium) The requirement 'Must provide utility to the target user' is non-specific and cannot be translated into testable or measurable product features without further definition of 'utility'. [req_001]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 3/5 — It correctly identifies the core actor (Cyclist) and SoI, but misses key domain entities like 'Bike' or 'Route'.
|
||||
- **model.accuracy:** 5/5 — The model is structurally sound; associations and blocks are used appropriately given the minimal input.
|
||||
- **model.parsimony:** 4/5 — It avoids padding with filler entities, keeping the scope tightly focused on the immediate inputs.
|
||||
- **constraint.capture:** 3/5 — It captures 'Undefined Scope' but fails to surface any meaningful non-functional constraints (e.g., performance, security).
|
||||
- **asm.detection:** 4/5 — The assumption is specific ('adopt and use') and tied to the vague scope, pushing back appropriately.
|
||||
- **risk.detection:** 5/5 — It correctly flags the vagueness of the scope and lack of defined interaction mechanisms, which is critical for this seed type.
|
||||
- **voice.character:** 4/5 — The notes are professional and appropriately cautious, though slightly generic in tone.
|
||||
- **Average:** 4.00 · **Min:** 3
|
||||
|
||||
> This run handles the vague-seed difficulty well by focusing on process gaps (risks) rather than filling content gaps. A stronger effort would suggest specific domain entities.
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.92
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TPBDJkmm38NtVGfhsB90mu_W2etu4OWLke876Bc9Mw99uifE6OtGlFjrIwywWDZL9-xvZftSQ44fVOgLRaBkK311ZMxJ2lVPttDaWQFcd1fQ7dJe6thUXjn2Wr7fuCH5L8MF2k3kXujli3aPJbWyoQJMbtXrkJJrLeItatX1v_xKc_Z8kJnZ8bWyKNod4Xmkglq1BtEXNt0dTEhEVi7jgxVW2IXrPGU9KasYLVKPVl7mJhcy5Y5C2q056ceTwxNkj522w-lFZzKAZfRk32_gCKGdNDXnN3jE7PPGXnWCy0TgvaYOhIK6Hrd9D7Eq3LgK2VjNsFUEP6HsKs4aT8N5G0CWOcvxR8VVAVVfe-NzR73FMViq6RYfc0s-Z0Slv6mlytnh6GLhhoExpSJhS1SP_PmMe8H4zcXIPw95iCtD2OPBw8HKUo5u3Mt6E1Bg0tsje_uDP8wbr_zZyUPO8s-FdOJXokCmP5NTKFRsKly1)
|
||||
- **Notes:** EventStream is the SoI; Producer and Consumer are actors; Event Record and Backpressure Signal are first-class data blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Producer (actor), Consumer (actor), Event Record (block), Backpressure Signal (block)
|
||||
- Associations: _none_
|
||||
- Constraints: _none_
|
||||
- Requirements: _none_
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system must be able to communicate backpressure signals in-band, meaning the `Backpressure Signal` block must be integrated into the core data flow. [backpressure_signal]
|
||||
- (0.90) The system assumes that platform engineers will prioritize operational simplicity (single binary, no ZooKeeper) over existing industry standards like Kafka's ecosystem. [eventstream_system]
|
||||
- (0.85) The system assumes that the `Producer` actor can be made wire-compatible with existing Kafka protocols while simultaneously introducing a novel backpressure signaling mechanism. [producer, eventstream_system]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The requirement for wire-compatibility with the Kafka producer protocol (eventstream_system) while simultaneously enforcing a first-class, in-band backpressure signal (backpressure_signal) may necessitate complex serialization layers that compromise the goal of operational simplicity. [eventstream_system, backpressure_signal]
|
||||
- (0.80 · medium) The model lacks any defined mechanism or element to enforce the 'Go-native' requirement for the producer (producer), which is critical to achieving the desired operational simplicity and single binary constraint. [producer]
|
||||
- (0.90 · high) The system design does not account for how consumer lag or backpressure signals are observed by the producer in-band (backpressure_signal), making the core value proposition of the product unaddressed. [producer, consumer, backpressure_signal]
|
||||
- (0.70 · medium) The constraint of running on a single node up to 100k events/sec (seed) conflicts with the need for wire-compatibility with Kafka's distributed architecture, potentially limiting scalability or increasing operational complexity. [eventstream_system]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The system block 'EventStream' requires protocol compatibility and license properties, but no associations or requirements link these properties to any specific element. [eventstream_system]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 3/5 — It identified core actors (Producer/Consumer) and data blocks (Event Record, Backpressure Signal), but missed key system components like the flow control mechanism itself.
|
||||
- **model.accuracy:** 2/5 — The model is structurally weak; it lists entities but provides no associations or relationships to show how they interact in a streaming context.
|
||||
- **model.parsimony:** 4/5 — The model is appropriately sparse and only includes necessary elements without adding unnecessary 'filler' blocks.
|
||||
- **constraint.capture:** 2/5 — It failed to capture the critical operational constraints (Go-native, single binary) or link them as requirements/constraints on specific components.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly relevant and focus on core technical trade-offs (simplicity vs. standards), which is excellent.
|
||||
- **risk.detection:** 5/5 — The risks identified are substantive failure modes, directly challenging the feasibility of combining Kafka compatibility with novel flow control.
|
||||
- **voice.character:** 4/5 — The findings are highly technical and skeptical, focusing on architectural conflicts rather than generic business concerns.
|
||||
- **Average:** 3.43 · **Min:** 2
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLN1Rjj64BtpAmREGQrWgQwT18aG64cA5WbGr4qGv4eCTuVKmiiTTdSfXqaDz2FoXVcIpfAKGz5MUv74cTrvRzxCerw6Y3wsjSt2jN4DUgoXG7LTUMwT_eKjUtXKFg6IpYSdmXOrtnXNGOasq2IZB8O0NpA0r_Ugw1BBivHXhycFqU8PFdzsBj5NtlFD67nAJ_I5bk1lxE8VM1ESl26xesWKdcIt4pp7aOx0dT65Udm4xZRR6ar0THCxgAako8Si6yYV_4aHFdXqmM9aVm8OeA6OnxiGl7WHkX2flho4H-VgCJujUWxOQiEvK6jZ7g8duRS1-I52INOdYCUpL7CqxD1A3YFbDMkoSg1WjeHE64_ONTqu-awCqvDmGYNSX5AYiiMGQwuc1OQOmdYiw4TMQCrdJ3JkQkycGO4eBAlh4MBeLcF4QH-bfxzQSghBGqCg-dRIvG7SQzOT_8wkQh6YExn2ehaTepF0GS0vOCE1yclg6ZOkXWV0thf8ll4KXoQksYWJw07DGIRdFZC3dSr5wfDhwo-DnQvixQbglU4sdABxr7yhMLXoyLPkXhQgIBP4vxLedSmouROdT-LMcdP6aP0foOjm1ALiMEFD3bKtw3By5crakz5tmb1Wf1HA0oFDLzRAELOKGaAgsn21bR03coPeEv3MuvPWH4EdGQ4J-q3RM4ODMw5FMxPw6EnFlTgGr5v_r_8DEsa9KAZ4mUkc54_smku2x2sotw3-MOpQxqF3rgW7oNyGNkmAHgyJ-R6FeGdwrCWr4wK3RSfUecZI80gADqGE5kE4PdGMVOkB2QF5Uf64dpZcxzLgxkmDY0eXcqLNgylxRjZiYHu_VxZD6w64jljy__nIY3oyZ1keIBXAmt9flWVhzULz_jY8O1JooDb3yl_VdSoI86hmfjf6cSro-LuCRJpL8dnOBeVGh-zMPsS_BvVmxP-lmzie255P31cBLDfXOUaqIIaUcNK6NAP5D3xHM0Guq3JDB8a6Uz7M4xpp4I-jo_3-7LV4XTRJtYPGT5BPYudgj4oo-nDh7PXWMllYI6svAeLI9zBXiFQfWCEO8AKL8--SSLO__ASXVbndekkWRlPIFkHFzry0)
|
||||
- **Notes:** Pet Translator is the SoI; Pet Owner is the primary actor; Vocalization and Body Language are first-class inputs/blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Pet Vocalization (block), Pet Body Language (block), Interpretation Output (block)
|
||||
- Associations: pet_owner→pet_translator:uses, pet_translator→pet_vocalization:processes, pet_translator→pet_body_language:analyzes, pet_translator→interpretation_output:generates
|
||||
- Constraints: On-device inference for privacy, Honest about classification vs translation
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that pet owners will find the generated interpretation output (interpretation_output) believable and useful enough to satisfy REQ-001. [req_001, interpretation_output]
|
||||
- (0.85) The model assumes that pet owners (pet_owner) will actively use the Pet Translator system (pet_translator) despite the technical complexity of providing both audio and video input streams. [a1, pet_owner, pet_translator]
|
||||
- (0.80) The design assumes that the combination of audio data (audio_data) and video data (video_data) is sufficient to accurately infer emotional state, making the combined input streams necessary for pet_translator. [pet_vocalization, pet_body_language, a2, a3]
|
||||
- (0.90) The system assumes that the technical feasibility of running all inference on-device (privacy_boundary) will not compromise the accuracy or performance required to meet REQ-001. [req_001, req_002, privacy_boundary]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The on-device inference constraint for processing both audio and video streams simultaneously may lead to unachievable latency targets or excessive battery drain on consumer-grade hardware. [pet_translator, privacy_boundary]
|
||||
- (0.85 · high) The model does not specify how the 'suggested_meaning' in the interpretation output is generated, creating a risk that the output will be perceived as random or ungrounded if the underlying logic is weak. [interpretation_output, req_001]
|
||||
- (0.75 · medium) The system relies on the pet owner providing both audio and video streams simultaneously; if one stream is missing or of poor quality, the combined analysis may fail to meet the 'believable' interpretation requirement. [audio_input_stream, video_input_stream, a2, a3]
|
||||
- (0.95 · high) The 'emotional_state_model' is a critical, undefined component that could fail to accurately classify nuanced pet emotions, undermining the core value proposition of the entire system. [pet_translator]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association 'a2' states that the system processes pet vocalization, but the `kind` of the associated block `pet_vocalization` is merely a data container and does not represent an input stream. [a2, pet_translator, pet_vocalization]
|
||||
- (0.85) The association 'a3' states that the system analyzes pet body language, but the `kind` of the associated block `pet_body_language` is merely a data container and does not represent an input stream. [a3, pet_translator, pet_body_language]
|
||||
- (0.75) The system's properties list `audio_input_stream` and `video_input_stream` on the `pet_translator` block, but no associated input blocks are linked to these specific property names. [pet_translator]
|
||||
- (0.70) The requirement REQ-001 promises that owners get a believable, useful interpretation of their pet's vocalization and body language, but the `interpretation_output` block only satisfies this without explicitly linking to the input data sources (vocalization or body language). [req_001, interpretation_output, pet_vocalization, pet_body_language]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actor (Pet Owner) and necessary inputs/outputs, but missed key domain entities like 'Veterinarian' or 'Pet Type'.
|
||||
- **model.accuracy:** 3/5 — Using generalization for a primary user relationship is questionable; classifying input blocks as data containers rather than streams would improve accuracy.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids excessive filler entities, keeping the scope tightly aligned with the core problem domain.
|
||||
- **constraint.capture:** 5/5 — Both explicit constraints (on-device, honesty) are perfectly captured as both dedicated blocks and requirements. This is excellent.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., technical feasibility vs accuracy), moving beyond generic user adoption platitudes.
|
||||
- **risk.detection:** 5/5 — The risks identified are high-severity, domain-specific failure modes (latency, ungrounded output) directly tied to the core technology.
|
||||
- **voice.character:** 4/5 — The findings are highly structured and technical, maintaining a skeptical, critical tone appropriate for feasibility assessment.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a very strong run; the risk detection was excellent, particularly in flagging technological failure modes. Minor improvements could be made to model relationships.
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLJ1JXin4BtlLup21I8bIWD5j48H557W02sqzzMiFRiPnMiljfUK8gH-HB-mNzBPtIIa9P6u9D7pp7lZUIy-2H5zh4kJX1dR2ZsMaA6Q5TxLLfyvuppivEyff_5QHPYYTdEs1UHe0gsTA8CXm5C2yEaLYyun7pKdpclo2pGxmWz7Oq5FlNVp1NX8x_M-5l2pi_46IuBU9Pa7YgombpolwLaNQOlSYFRLmHQvvsJAce3AAZv2IML6FYH9DtpljcQASEbg7-3ROuXKzW03t3TmEctWDBGm71ztFoOJs1ch0ppCbXnNLlC3wneDdCaTF6TrTBxbuTLHgbwEX0kLV4ycIufpMx0byisQhz5YGLq_lU1fsU6lcy_GaDNeuVJhLTkb5a2A5KjvPfoQlPHVa-nMTExelgOGsuwomrBVOMjDowuxGZsOUnOBBfmh3C5A5gs6BtKqpirWzvgLTy7b4IuyLjCzMK6amcDaPrifBqGjJxfs8eAodn0z2YWRVXBNbkFMCe-dK96A4BcadyuIQCvpycGLoP9-L8OLH_CeOb1vLxaWKMtLbWtfac6xq8q357TA6KX1U_rcItEE4eC8TP3LIslElkn6piN8E2KGH2RS7u76nm2xMSr6G_GozjuY0nnP0j12QILcEhrfZ6HBM0Q3oOOy9fkYiQLqGvhxWy4_CKd-Jy0Mgitk3OURYk4ZO2NMK4YZIruvyPQcZRjxIsDYg93BWUTY6kLXwFURkBEdacmC_Nu7dTyEHgDt_Jxy-VKRlekN2wgooHpPK7jgBMxDzsGOCmbPGJ40HZ24KZPkKX6vg5qTG1DLWxdpiov00MHac1FDXck2uos2gAPC3uqYXrNkb0ljltKrHkKCDs-Z4VygZcY64WNTNJSvaGzvuFy2)
|
||||
- **Post-validate:** stripped 3 refs, dropped 1 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours System (system), Individual Contributor (actor), Engineering Manager (actor), Calendar API (block), Meeting Request (block)
|
||||
- Associations: individual_contributor→quiet_hours_system:, engineering_manager→quiet_hours_system:, quiet_hours_system→calendar_api:, meeting_request→quiet_hours_system:
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The tool assumes that negotiating protected blocks invisibly (proposing alternative meeting times) will successfully prevent the user from incurring social cost or political capital loss. [req_002]
|
||||
- (0.85) It is assumed that individual contributors and engineering managers are willing to adopt a tool that proactively manages their calendars without explicit manual intervention. [individual_contributor, engineering_manager, a1, a2]
|
||||
- (0.80) The product relies on the assumption that existing corporate cultures are sufficiently receptive to a tool that attempts to manage and negotiate meeting schedules on behalf of its users. [req_002]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.85 · high) The model does not specify how the system will handle conflicts when a proposed deep-work block overlaps with an existing, high-priority meeting that cannot be rescheduled. [quiet_hours_system, meeting_request, calendar_api]
|
||||
- (0.90 · high) The system's ability to negotiate protected blocks invisibly relies on the 'Calendar API' having sufficient read/write permissions and intelligence to interpret meeting priority and attendee availability across both Google Calendar and Microsoft Graph. [calendar_api, readwrite_integration, quiet_hours_system]
|
||||
- (0.75 · medium) The 'initial_trust_period' constraint requires the system to build trust, but the model lacks any defined mechanism for measuring or improving user trust, making successful adoption uncertain. [quiet_hours_system, initial_trust_period]
|
||||
- (0.80 · medium) The model does not account for the complexity of coordinating deep-work blocks across multiple time zones and different organizational cultures, potentially leading to proposals that are impractical or ignored. [timezone_respect, quiet_hours_system]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The association `a4` from `meeting_request` to `quiet_hours_system` is semantically incorrect because a meeting request should be an input or trigger for the system, not merely associated with it.
|
||||
- (0.85) The requirement `req_002` (achieving focus without costing political capital) is an over-broad claim because the current blocks lack mechanisms to model or enforce 'political capital' or social cost.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It correctly identified the core actors (IC/EM) and system components (Calendar API), but missed key concepts like 'meeting' or 'deep work block'.
|
||||
- **model.accuracy:** 3/5 — The associations are too sparse; while composition is used for the API, more explicit relationships between meeting requests and the system are needed.
|
||||
- **model.parsimony:** 4/5 — The model is appropriately focused on core components without adding unnecessary filler blocks or entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were captured accurately and applied to the system block where they belong.
|
||||
- **asm.detection:** 4/5 — The assumptions are specific (e.g., negotiating invisibly) and tied directly to core requirements/actors, avoiding generic platitudes.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (API limitations, conflict handling), and tie back to explicit constraints or system functions.
|
||||
- **voice.character:** 4/5 — The findings are structured, critical, and specific, maintaining a professional, skeptical tone without being overly verbose.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run; the risk detection was excellent. Improving model coverage by adding 'Meeting' or 'Deep Work Block' entities would make it near-perfect.
|
||||
|
||||
---
|
||||
403
phase-0/reports/2026-04-28-2115-run.md
Normal file
403
phase-0/reports/2026-04-28-2115-run.md
Normal file
@@ -0,0 +1,403 @@
|
||||
# Phase 0 corpus run · 2026-04-28T19:15:17.539Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 8 · **OK:** 8 · **Failed:** 0
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 4/3/3/2 | 2/4/3 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 02-habit-coach | medium | 5/4/3/3 | 4/4/3 | 0.85 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 03-redline-ai | medium | 6/4/4/2 | 4/5/4 | 0.92 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 04-skillswap | medium | 5/5/3/3 | 3/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 05-cyclist-thing | failure-prone | 4/3/0/1 | 1/1/3 | 0.30 | 3 | 4 | 2 | 3 | 2 | 4 | 3 | **3.00** | **2** |
|
||||
| 06-eventstream | hard | 5/4/3/3 | 4/4/3 | 0.92 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 07-pet-translator | failure-prone | 4/3/2/2 | 4/4/3 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 08-quiet-hours | medium | 5/4/3/3 | 4/4/3 | 0.95 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
|
||||
**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed
|
||||
|
||||
**Passing:** 7 / 8
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/bLNRJkCm47tFLrZbhLAgKYvYKOLW4MXVMB70FbkECqczTUoinsbLeKhx4Vk5VCcEdRG4Q97s1OcvdJFdJDnBzCBvfj89pfMfXHCLouISbyus9h-ssZfsK9n00PDU1Sv4RfVAbAmG6g6NaLeWikU4iQyVfkI5ACOXOrqEhejcP-BBsOIYLyxPPHSyXPFyAAVWhJN-kwY03Rw1Ne1NKWoITG_FM0zxuCPm98_tmAsJcSg1GLNx5QkWoi1XahJa1rTEeRTUmu09P6Bp7vjESOKUgeiBTZ2Hn-8qYyW8YCeQFWjrRiNE6Nf7sb2gGN2yThPGrBwDhpT0FmqHAvt86q5RFFec1-CZQDFFS6mpH418WY3yW2jjun0u66UrhaA-XooaoA5IacjOWAOCcAPwBfH3dwv0kFKhY-iuPsdTdDqHcbD2Hmvo6-TL5oS6cRPotZ709li5qdFXGFJ1QUDCGyKH0gVCMWt2l08-pPIPX-EvfzCpuDXZgs14zLsIrssIT_A-WrQ6694QGNXjotVePEj7EzPRW-yQzCoGC8xPnjUDPuAshMeDP0DQtOInp5iA1myWtsmK6LRKpMCtRxlvjcKwbTOGd21waUXsvmV0cbB0D4bfvAfrc1sE9iZkJyO1h7P0vx80LcYxZ4XTBQ-fOF_esvk7-oj6bIBCVVdpj-TSjoNflm0aqT0h7wcIbrduM8HRHQW2N2tuPiWEi6JtIQRfHUyBET_wWdof_2nvJON2NHTr7hILv37W7eXXckw_2cgdY9AgfXRaoYIVs3CQlIKhwbehwl8swKlzITbRcVwhSFVolPP4GmacPqwLCqztF1m-mEz6EGZVDWw7RUZcHpeU7mw7qUYd6TrhV9FYLT9hB9LMFhmcT4CflUrpijr12gOKPHp4WYzims3K6phPCxHimbFfGxHp98mWRODp2vN7dm4LBceZD8uIMgHT9xcaF_JxyWy0)
|
||||
- **Notes:** Aristotle is the SoI; undergraduate_student is the primary actor; coursework_material and thinking_partner_session are key blocks.
|
||||
- **Post-validate:** stripped 2 refs, dropped 1 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Undergraduate Student (actor), Coursework Material (block), Thinking Partner Session (block)
|
||||
- Associations: undergraduate_student→aristotle:interacts_with, aristotle→coursework_material:processes, undergraduate_student→thinking_partner_session:participates_in
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.85) The system's refusal to provide solutions will be perceived by students as helpful and engaging, rather than merely frustrating or unhelpful. [req_002]
|
||||
- (0.75) Students are willing to use a tool that requires them to input their coursework material for processing. [undergraduate_student, coursework_material]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The 'FERPA tenancy' constraint requires that coursework never leaves the institutional boundary, which may conflict with using external LLM APIs necessary to achieve the low-stakes thinking partner function. [ferpa_tenancy, aristotle, coursework_material]
|
||||
- (0.90 · medium) The 'latency_p50' constraint of 1.2s may be unachievable if the system must process complex, multi-turn interactions involving detailed analysis of 'coursework_material' using a large language model. [latency_p50, aristotle]
|
||||
- (0.85 · high) The 'must_never_output_solution' constraint is difficult to enforce programmatically across all possible student inputs without introducing complex, brittle guardrails that degrade the natural conversational flow. [must_never_output_solution, aristotle]
|
||||
- (0.70 · medium) The model lacks a defined mechanism for how the 'undergraduate_student' accesses or submits their 'coursework_material', creating an unaddressed dependency in the user journey. [a2, undergraduate_student, coursework_material]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) Constraint `must_never_output_solution` applies to the system `aristotle`, but this constraint is more accurately a functional requirement that should apply to the output generation process, not the system itself. [must_never_output_solution, aristotle]
|
||||
- (0.75) The association `a2` (from `aristotle` to `coursework_material`) is labeled 'processes', implying a directional flow of information or action, but the system's role in processing coursework material is not explicitly defined by any associated requirement. [a2, aristotle, coursework_material]
|
||||
- (0.70) The block `thinking_partner_session` is defined as a 'block' but has no associations connecting it to the core system or actors, suggesting it may be unused. [thinking_partner_session]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It captured the core student and material entities but missed key actors like 'Professor' or 'Course LMS'.
|
||||
- **model.accuracy:** 3/5 — The associations are generally correct, but classifying `thinking_partner_session` as a composition is questionable.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids unnecessary filler blocks; it only includes necessary components.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly captured and applied to the system/blocks.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., perceived helpfulness) and tied directly to core requirements.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific, and challenge the feasibility of constraints.
|
||||
- **voice.character:** 4/5 — The notes and findings are clear, structured, and skeptical without being overly verbose.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.85
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRZjGu47tFLvI65sXDZsPd0C48ZM07M2HOQ1XujYfE9IcrOmVRwQONZRGViL-uNx9bftixmrmUkXENxJgdwXmxBqD47uVE5679jaUF7PIeburtWwqkd74U7jHFgAQJoOhGOkNMR1keqGIQp6Y38S3F0k3t6rcg6klZDEDyHNuRBP_XysSd4dtblLjlWq_fINLQIV2ji_4ZTWG7xyYiABB6W-9gWcTTf3lWZkbKFxu3xgfekIAWheyRwAWhoOUY6CaVlCEI8rmurEq1O80sZPLEOtZn8cn2fExy71wSwCVujCpe5KPKs3IU6eo2SmQbSuRGofodtlaepL9i8_aQDKtchtQe2_HaOmRiywjWePPa0fKWHAU6eXe2fVGXUiaeOIqTyiueGoioCa5sw7wstBI77LKyT8V6hQ_sA1Sjcog3wFIsnzWjkDIUoC9NbgQ_bghosf23AafGfL9bMscSNcwf3G4RKXq99nscvB3lbSP8ZVERVNnFnjcLD1uZEmiBxqIDaTeahleS_mKnkfwraZRX52uCJKEYJgM-3_9CMIUWEzG_Xn1XxVmIq3ZRX6G4_dG9qyhHLl27SuqXU8iTcmryOBiyJ7PBUfgaOMofotGxIIZse1yippN7LhcWekWRXBNK8WnGfC_KhmkHVPFtGqbRsUTRHGrLq8lM5vyMYrVmyDlbev7LwyMN-Nju-ERRexsCMHJf5AzGR-u1_UWWECree2QCWwS0jRXroLM0V__-Hye3CQMSbjAXhqAg5_gXDApJaKzzp9ZMgJ6D-Z_DBPZ5riRp-Vcr8tIMzAsvIVkKuOvZK2l6OZJapTKhfdKOAsHRJEVIsXlMF8FTEPJqqZ7veTdyHU76rjjiTWODMV9vixIck4VRewDVcOhT3LEQBswHvTxrjxJqtlM58I5QWPUJ7iNwizaN-ZwmfqvQ7cQpCVJcy_pu-BVPB8lxjIKOBw_HSN8HiU4efGA2t9noHT5XrbcSD-sDU3Ekqz7O2fficEixcg2SJ52kRTwvUxoS9JNR7KZMIzp6AndK8i1MJKXgZhVA5EBqZaB8OcdoUTnUQj5bqupv4riXBfyExbEj0Y-EW-oAmE6ea3QEpInUofzy2ly3)
|
||||
- **Notes:** Habit Coach is the SoI; Parent and Child are actors; Screen Time Data and Conversation Prompt are first-class blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (actor), Screen Time Data (block), Conversation Prompt (block)
|
||||
- Associations: parent→habit_coach:configures_limits_via, child→habit_coach:views_data_in, habit_coach→screen_time_data:aggregates_and_analyzes, habit_coach→conversation_prompt:generates_for
|
||||
- Constraints: Must work alongside iOS Screen Time and Google Family Link, not replace them, Child data must be parent-controlled per COPPA (US) and GDPR-K (EU), No social features for kids — no leaderboards, no public sharing
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that parents will find the data aggregation and analysis provided by `habit_coach` sufficient to facilitate a routine, shared conversation (REQ-001). [req_001, conversation_prompt, habit_coach]
|
||||
- (0.90) The model assumes that the child will actively view and engage with data presented by `habit_coach` (a2) to make the conversation routine, rather than simply viewing it. [child, a2, habit_coach]
|
||||
- (0.85) The system assumes that the data aggregated by `habit_coach` (a3) will be meaningful enough to generate actionable conversation prompts (`conversation_prompt`) for both parent and child. [a3, screen_time_data, conversation_prompt]
|
||||
- (0.80) The initial hypothesis assumes that the mere visibility of data to the child (`child` actor) will be enough to shift the parent's role from 'enforcer' to a collaborative partner. [child, a2, habit_coach]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on the `habit_coach` block to aggregate and analyze data from `screen_time_data`, but the model does not specify how it handles raw, continuous usage metrics or potential data gaps (e.g., when a child is offline), risking incomplete analysis. [habit_coach, screen_time_data, a3]
|
||||
- (0.85 · high) The model assumes the `parent` actor can configure limits via `a1`, but it fails to define how the system reconciles its own limit configuration with existing OS-level tools (iOS Screen Time, Google Family Link), risking conflicting enforcement signals. [parent, a1, must_coexist_with_os_tools]
|
||||
- (0.90 · medium) The `child` actor is shown to view data via `a2`, but the model lacks a mechanism for how the child interacts with or influences the generation of the `conversation_prompt`, risking the conversation being perceived as purely parental critique. [child, a2, habit_coach, conversation_prompt]
|
||||
- (0.85 · medium) The `habit_coach` system is tasked with generating conversation prompts (`a4`), but the model does not specify how it ensures these prompts are genuinely 'shared' and collaborative, as required by `req_001`, risking a one-sided communication dynamic. [habit_coach, conversation_prompt, a4, req_001]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association `a1` (parent → habit_coach: configures_limits_via) implies the system allows parents to configure limits, which contradicts the constraint `no_social_features` if 'configuring limits' involves any form of public or shared enforcement mechanism. [a1, habit_coach, parent, no_social_features]
|
||||
- (0.90) The association `a2` (child → habit_coach: views_data_in) suggests the child is the source of viewing data, which contradicts the constraint `child_data_privacy` that mandates parent control over all child data. [a2, habit_coach, child, child_data_privacy]
|
||||
- (0.80) Requirement `req_003` states the system must enable the child to view data and participate in its analysis, but this requirement is only satisfied by linking the `child` actor and the `habit_coach` system without specifying how parent control (from `child_data_privacy`) mediates that viewing. [req_003, child, habit_coach, child_data_privacy]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model correctly identified the core actors (Parent, Child) and system components (Data, Conversation Prompt).
|
||||
- **model.accuracy:** 4/5 — Most associations are correct, but 'views_data_in' is weak; parent control should mediate child viewing.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and avoids adding unnecessary or fabricated blocks/entities.
|
||||
- **constraint.capture:** 5/5 — All three constraints are explicitly captured and correctly applied to the system block.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., data meaning, child engagement) and tied directly to core functional elements.
|
||||
- **risk.detection:** 5/5 — Risks address high-severity failure modes like OS conflicts and data gaps, showing domain depth.
|
||||
- **voice.character:** 4/5 — The findings are structured, specific, and highly technical, maintaining a skeptical PM tone.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.92
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNTRXCx4BxVfnYL6uYQgeGVGOGgIiu1Al4Ze1ANrgntTkFMQo-sDpaHgdGUWYVaIHYlTvCjQNeJnVFpVUFvPholVK0Nsbfd_bgP1XtMaAEyhfnjJR6msZfuK3wZacQZ2B_4mgwLgQ14xMdaaHgzXvyPm9izbAB4yZHwh2l8zTRy1RvyCMFhkNDstHkVqxFYIS76jzQ4JrWJ7BqdlQAW91vbDoC-OmCTe3kb9_Bf0RgRRAaA0ggRi86QwfoSpx9K_D5NAhGo1ESNHu0UN3eALF3gbT_uGFNP6JoOoQVuFE-ubGbKEGpA6j5e3ALrDSp11ySjOZ_Zjfw4Lhb3jn4e9J7F779hDQ7X26mB5OIsbV31EgnevBqPwlf05Mgu9AoxkdGyYi17hWibvt5PiHp7UPnFfgr_LcJ8SPZapdZInrUjKTO9nhDIOQ2R7Vo2U-qO1_wnigt9X8v5zbPHz5OcoxMLrprPz4VhwBf3mtYqGkk7IXeo1VkF7QqKhQauBfL1FURlkWGVKfSI-Qt6xJ7N3PgDI45-NC6AvMGnue3UqST1tszjLP6FgjtIsM_DVn7s3Yg4NABfDDeoNhMSKV9uTdaZHnIx2MZaeNwa0Ew7eokI_6VKaezAEkjj6U0x3t5NPJsOn9fDE-Mtq_kHAzbE9C5QXINSXSDXOXicOjY0o99o-SZExkBx0p9e7Z7047SRkdY8B7ugsMnhDjQs8EtXOUFKIl6uKioKv8n_rDLWhEWon90XEfWes-4gpkEgT7mQR0cHVGEz82FfkjfoNl7_6ki2vfewojrHf9rBG8VfldrUm0mkDmt1nKKtSrf5DQ5qjeO2Ds1DWkQDced0WK8fCGgy0piRxUvqUZP-OkOGXrdm4oBIfcGZPupTNyWvDCx69uNyNl3j1Ph7RD_guBDzcCbqUjSEpDaeIQsug9tLPmVdv-Jah_jWmvtWx62pUmI_B-s0QFUb71BZlfoe2YybE5Kj0w_kPFALVhJAKMoFdqoIwTylqzFJnvC9_Fx_5xnLpeTf4oUbH_Ky3xpzQ0haPWnTJlkMneTuAtbNQcNQGFvaX3hhKI-N1E_ImmsBz733mtUBHmaaZGL_dKRZxXcHNoOiuafW6vOsFZ-xchfXzPAbaikJZ2-PhfgzvX_--Fy1)
|
||||
- **Notes:** Redline AI is the SoI; Legal Team and MS Word are actors; Contract Document, Clause Library, and Redline Suggestion are core blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Clause Library (block), Redline Suggestion (block), Microsoft Word (actor)
|
||||
- Associations: legal_team→redline_ai:uses_for_review, redline_ai→contract_document:processes, redline_ai→clause_library:consults, contract_document→redline_suggestion:receives_suggestions
|
||||
- Constraints: Must integrate with Microsoft Word, Contract content must never train cross-customer models (privilege concerns), Audit trail of every change suggestion must be exportable, SOC 2 Type II compliance from day one
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that the legal team will accept AI-generated suggestions for first-pass redlining, even if they are not perfect. [a1, req_002]
|
||||
- (0.90) The product assumes that the legal team will find AI-generated redlines significantly faster and more reliable than a tired junior associate. [redline_ai, req_001]
|
||||
- (0.85) The system assumes that the legal team will trust AI suggestions enough to use them as a starting point for review. [redline_suggestion, a1]
|
||||
- (0.75) The product assumes that the legal team will be willing to adopt a tool that requires integration with Microsoft Word. [must_integrate_with_word, microsoft_word]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's reliance on the `microsoft_word` actor for core functionality creates a single point of failure if Word integration fails or changes. [must_integrate_with_word]
|
||||
- (0.95 · high) The `no_cross_customer_training` constraint requires a robust data isolation architecture that must be explicitly designed into the `redline_ai` system, otherwise privilege concerns are violated. [no_cross_customer_training, redline_ai]
|
||||
- (0.85 · medium) The `exportable_audit_trail` constraint requires the `redline_suggestion` block to capture not just the suggestion, but also the specific user/system identity that accepted or rejected it. [exportable_audit_trail, redline_suggestion]
|
||||
- (0.70 · medium) The `clause_library` block's properties only list `company_clauses` and `versioning`, but the system needs a mechanism to link specific clauses to their legal domain or risk type for effective redlining. [clause_library]
|
||||
- (0.75 · medium) The `legal_team` actor is defined with only a generic `role`, which limits the ability to model workflow dependencies between different roles (e.g., junior associate vs. GC) as required by the problem statement. [legal_team]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association 'a1' (uses_for_review) links the actor 'legal_team' to the system 'redline_ai', but the requirement REQ-002 states that the General Counsel must review AI suggestions, implying a specific role interaction not captured by the general 'legal_team' block. [a1, legal_team, req_002]
|
||||
- (0.90) The requirement REQ-002 specifies that the General Counsel (GC) must review AI suggestions, but this specific role is not modeled as a distinct actor or property within the 'legal_team' block. [req_002, legal_team]
|
||||
- (0.80) The system 'redline_ai' has a property 'audit_log_storage', but no block or association is defined to represent the storage mechanism for this audit trail, making the constraint 'exportable_audit_trail' difficult to satisfy. [redline_ai, audit_log_storage, exportable_audit_trail]
|
||||
- (0.75) The association 'a4' (receives_suggestions) defines a composition from 'contract_document' to 'redline_suggestion', which implies that suggestions are inherent parts of the contract, but this relationship should likely be an association indicating the suggestion is *applied* to or *related to* the document. [a4, contract_document, redline_suggestion]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core system (Redline AI), key actors (Legal Team, MS Word), and necessary components (Contract, Clause Library).
|
||||
- **model.accuracy:** 3/5 — Using 'composition' for suggestions applied to a document is inaccurate; associations are better suited here.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids excessive filler blocks, keeping the scope tightly aligned with the problem domain.
|
||||
- **constraint.capture:** 5/5 — All four explicit constraints were correctly captured as dedicated constraint blocks applied to the system.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., trust level, speed improvement) and tied directly to functional requirements or user pain points.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (data isolation, single point of failure), and tie back to constraints/elements.
|
||||
- **voice.character:** 4/5 — The findings are structured, specific, and skeptical, though some assumptions could be framed more as risks.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run; the risk detection was excellent, identifying critical architectural concerns. Minor improvements needed in modeling relationships (composition vs association).
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNRRXit47tVhnWuBuaW6QwS1AaG66cR50sG16rSe8-B4JcxIuXBRaYkPC4mq8_e5_PBEYHNybgN8Y_sQZWpvypC6VAT3-X2t-h2hvJfq64BInIhsjdUo5-ijWwULQ-eeld8mpSexKQP6YhKdaOdGgFtS5y0_7oKHLPONSKJwoIvmRfyWp--cRFr9-ViPZ2-fbVoMhBnLsl25sm9BduZlQQW15uK3oCyOmETWRkYQ_7o3Dn3qIX9G6qNjj1IkoJdYoAJlxXTAQrlDzZ1xnf3PLrx0UZ1HxDdSzaDPdZxrczzeFRc1fxDnKjylKnSw4uqQ6eg6-M3TLjOW0-EsyLdBGRHy6U9khPEXQPzF7tOCVXARQphB-jMv1AuIvPoaooCYu8JC-nzIid6W4jDPUOundFKzG63igRqmZhYCzF7UX_HKhtmYRboo6EffSuMHbjgArO3MZuBsux6C9oxutfjx_mZ0e0aBvpg8lo9CZyCVO9RyfvT4lQ-UJuR3-6ZNikWsYVmPEIHhIAIKSgbmrJ6WhtwzhwprarnJGvhcjRMoYaZajPamiDXJpx7OOqwaePtG2ZPI9veVN2z3yTTtq5yiT1QGrl8zNhWVzA3CZvACpG4tLX-nfR9lGmEZQ-I2eGrZ8RAX4Hcp_wZv6eKIvlBL1KlJHh0MY5FsQn8VZJ0o-1LgpGwmC2VjUcxXAE6sF99x7cizygpmfY-mAuZ2O7B8gcGzSdYMolQmBz__yFqOvskfZWIe6-zMcFiUSAKEKUP0y_W5KztO3QxERMI2-XPEFlV1q6JsUn0t0iGZf3v5oUIxJ66oI-W9iFAsRiFzkZt56W1dRETZLGuqwdSaSgHrXVGM1zySJpioyipvNBNjMAOO8lJeukHX-H6GSUz_nweGXEtGe9JTHDOwvF9rpXJnc4AVZB9fWz_pAwkVfXCaWB-72aQMbuDVZo4qefl9ueBF0js5bCAstQQ4XDR0MOg4Bd0yztjEKqNJlhOzN0AUSzVm8PlLSN1Xc13jBeSSPcFkEIhEpCXutiNbuvt8MwUCYeer0Fs3WAKpnUT15vLMCRNXh-PrHtCP_7osxlw5sFOw_-7NThG0BzdWPiRgyZRlqTb6J6RDUbjNAwXFTadTKLiXQRBWdkNXrAyup_yfFy7)
|
||||
- **Notes:** SkillSwap Platform is the SoI; Remote Worker is the primary actor; Skill Listing and Exchange Session are key blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: SkillSwap Platform (system), Remote Worker (actor), Skill Listing (block), Exchange Session (block), Skill Match (block)
|
||||
- Associations: remote_worker→skillswap_platform:uses_platform, remote_worker→skill_listing:creates, skillswap_platform→skill_match:generates, skill_match→remote_worker:proposes_for, skillswap_platform→exchange_session:hosts
|
||||
- Constraints: No money changes hands inside the platform, Identity verification via LinkedIn or similar at signup, Disputes capped to mediator review — no chargeback equivalent
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The platform assumes that remote workers will use LinkedIn or similar identity verification methods for signup, which is necessary to establish the 'reputation_score' on the 'remote_worker' actor.
|
||||
- (0.85) It is assumed that a high-trust reputation ('reputation_score') can be successfully ported from external sources (like LinkedIn) and used to drive successful matches via the 'skill_match' block.
|
||||
- (0.75) The model assumes that the primary mechanism for completing a swap is through the generation of a 'skill_match' association, implying that matching success guarantees user engagement with the subsequent 'exchange_session'.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The model does not define how 'reputation portable from LinkedIn' (initial hypothesis) is translated into the `trust_score` property of the `skill_match` block, creating a critical data gap. [remote_worker, skill_match]
|
||||
- (0.90 · medium) The `matching_algorithm` property on the `skillswap_platform` block is insufficiently detailed to guarantee high-trust matching, potentially leading to poor user experience and churn. [skillswap_platform]
|
||||
- (0.85 · medium) The `remote_worker` actor relies on external identity verification (`identity_verification`) but the model lacks a mechanism to handle or enforce the revocation of that identity, posing an ongoing risk. [remote_worker, identity_verification]
|
||||
- (0.90 · medium) The `skill_match` block only includes a generic `trust_score`, failing to model the specific, reciprocal nature of skill exchange required for successful swaps. [skill_match]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-001 requires tracking 'matching, scoping, exchange, rating' end-to-end, but the `skill_match` block only contains properties for 'matched_skills' and 'trust_score', omitting explicit scope or rating data. [req_001, skill_match]
|
||||
- (0.85) Requirement REQ-002 mandates that the initial skill exchange be scoped and bounded (max 2-hour exchanges), but this constraint is only satisfied by properties on `skill_listing` which describes a general listing, not the actual bound of an 'exchange session'. [req_002, skill_listing, exchange_session]
|
||||
- (0.95) Requirement REQ-003 requires both parties to rate the exchange positively to complete the swap cycle, but the `exchange_session` block only contains a single property for 'feedback_rating', failing to model distinct ratings from two separate remote workers. [req_003, exchange_session]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It captured the core actor (Remote Worker) and key processes (Listing, Matching, Session), but missed explicit representation of the 'skill exchange' itself as a central entity.
|
||||
- **model.accuracy:** 4/5 — The associations are generally correct (composition/association usage), though `a3` (platform generates match) might be better modeled as an association between two workers via the platform.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and only includes necessary blocks and properties without adding unnecessary filler entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly transferred to the model's constraint section, maintaining fidelity to the seed text.
|
||||
- **asm.detection:** 4/5 — The assumptions are specific (e.g., linking reputation portability to skill_match) and tied directly to the initial hypothesis/constraints, avoiding platitudes.
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, focusing on critical failure modes like data gaps (reputation translation) and process failures (scope definition).
|
||||
- **voice.character:** 4/5 — The findings are structured, specific, and critically analytical, maintaining a professional, skeptical tone throughout the report.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run; the detection of inconsistencies regarding rating requirements (REQ-003) and scope definition (REQ-002) shows deep domain understanding.
|
||||
|
||||
---
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.30
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TPFTQjj048NlzHI3SoVYu3e_f2QOf6bBRrfewBqOxOwbnVkZxeuKr63eG_G9-oGPnOwZkCsD46VEpdTcjBfEZ95RPxEqDhx1Y0vALEigXjRhss13XAFLEQreFdAa6dMuDxw25Tf4eugoc18yP00V_kcYLxYQ3PKGDSMTMbxY-ykvg3SnXlkTU47d-bIB-3bu_eQEOFA5R4Ti54wopOZd0zCRk1cTghCtS9kiDfg0NCCzE78bnPHbs_2JsrvPanX-r3BX1321sYe53mfSNQK-CRdb4excwWmloeEJQNn8xAWuH74FEUNTT8RxWlk6O07aM_UWWdCjKpoEf28Xc-3H7aTKj96eu-Q2lmjYVI7ugeY38fpI1hNUSRIWqIigraueYQCOHIPBtLF_eaBZYr5jpxdP9OIFoFZ4sMVMeXnms3WgT1ktJPtn4YMDYTXHn8gAr13fOjtKOsJslAJfT7cmvGMqYLBsMjpRzbCl06LN_Q__Mby7Nu2iLow3V6bB5NaDqLGrIyOylwEVhOdao7FAywtqwVjqDdkNv_3tzn_usahA9ePkk3Sj6pjiY6jaC0awj2sMbe23QFGyoKaca2qgkvQ7_6QF)
|
||||
- **Notes:** The seed was too vague to model specific features; therefore, generic blocks (cycling_route, activity_data) and associations were inferred to provide structure. The overall confidence is low.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclists (actor), Cycling Route (block), Activity Data (block)
|
||||
- Associations: cyclist→cyclist_thing:uses, cyclist_thing→cycling_route:analyzes, cyclist_thing→activity_data:records
|
||||
- Constraints: _none_
|
||||
- Requirements: REQ-001
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The product will be useful enough for cyclists to warrant adoption and sustained use. [req_001]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system lacks a defined mechanism for how 'Cyclists' (actor) interact with or generate data for 'Activity Data' (block), making the core function of recording activity questionable. [cyclist, activity_data]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association 'a2' implies that the system analyzes a 'cycling_route', but no requirement mandates or suggests this analysis is necessary for utility.
|
||||
- (0.75) The block 'activity_data' has properties like 'time_duration_minutes' and 'average_speed_kph', suggesting it represents measured data, but no requirement mandates the recording or use of this specific data.
|
||||
- (0.70) The block 'cycling_route' has properties like 'distance_km' and 'elevation_gain_m', suggesting measurable metrics, yet there is no explicit requirement to calculate or display these values for the user.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 3/5 — It captured core elements (Cyclists, Activity Data) but missed key functional areas like navigation or community features.
|
||||
- **model.accuracy:** 4/5 — The relationships are generally sound (uses/records), though the 'analyzes' association is slightly vague and unconstrained.
|
||||
- **model.parsimony:** 2/5 — It added generic blocks (cycling_route) that aren't strictly necessary given the vagueness of the seed.
|
||||
- **constraint.capture:** 3/5 — The single requirement is too general; it fails to capture specific functional or non-functional constraints needed for utility.
|
||||
- **asm.detection:** 2/5 — The assumption ('will be useful enough') is a generic platitude and doesn't challenge the underlying ambiguity of the seed itself.
|
||||
- **risk.detection:** 4/5 — It correctly identified the core data flow gap (Cyclist -> Activity Data), which is substantive and domain-specific.
|
||||
- **voice.character:** 3/5 — The notes are appropriately self-aware about vagueness but sound more descriptive than skeptical or question-led.
|
||||
- **Average:** 3.00 · **Min:** 2
|
||||
|
||||
> The model correctly identifies the limitations of the seed's scope by flagging multiple inconsistencies and risks. However, it overcompensates by adding generic blocks, which detracts from parsimony.
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.92
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XPNRRjim38Rl_HH4UXSqcPjsHLS4GRkYmu2rGxSEsAL1oxGZM9OyIKuGR0Ns47l2FSdeKtDeqjq442dnetx-SgwSH-khGWKkbxf4YmN4AFBCcaedDqOP2qVfMqffih73pJ4nIwapI54vsiW8XSx1pm3W_OigIOffM6UCJSXsqVW2trrCE7fjhLbsmNDwcvmc7FnWjF-C1S7W8wa5UIbm43njyBJnT00NqgauEu1x2kOo8Q2Yz2ieg8Z9kY1ecn_SBaZxHsy9Ym6W0whNhbd3TEfMpbCncy7HH9pXUTpGIsky4KP52ouaZOPBu0Ci4ETgEKjBpbMM8YSpZOhJiJ6AK7Cd7VN1cgGIP1jasIsO5oiZSiRLc8AhO4PHYIjbCFaF9MAHAh-FTMEqgugE9Rh53ejR45oEgqSMMUHBq5Mj4wSKPX7xnbTktS7JbdfmttQwrY_gUj-XU5DAiNcFd5QRomMgYlPG7XJwrDW2RdKcDVMoTT68kYZJK7XZPxFdwNwNbaR252Lw6Ik2fVHp-8HfZclL-t425uD2PYn0FLI7AVbLGyhh0z6IQqLzBQaacsawhVNq5gNspHsVMtxa2p0lbXhjwXXSk-JTgUHt91KTWwsq0mPXdzM6NLgLB1ESX67UMV6D8z5qqUwAsf9HGgKogwAM_t0NroMAES5a78AIWhHhPuHDE9f4OTI5zzK8NWey7iys7qYmTkwlqMWhi_RPRkRvpB1Fz6u93kgymsMBO5agIIxo9ZYeowkdzcZmwlv0KKOw0IkpkMUl3eTVwKV5hTOeDnoseTilep0y6GxXx-y_q3zo1qNbF5WI91UqzNYXVRokcGhxmZe_QcV7NpmF9WNFy-iTEjw0J3h8DywtNwcMab2AbVArYVePZ1BBN0sfCiiwwErsgTC3fUQe4xPbmRQq6kxkxaUbB4cn5H9O6flNNoKt3gZULMiJNF4F_xly0m00)
|
||||
- **Notes:** EventStream is the SoI; Platform Engineer is the primary actor; Producer and Consumer are key functional blocks interacting with Event Messages.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Producer (block), Consumer (block), Event Message (block), Platform Engineer (actor)
|
||||
- Associations: platform_engineer→eventstream:, producer→eventstream:, consumer→eventstream:, producer→event_message:
|
||||
- Constraints: Wire-compatible with Kafka producer protocol for migration safety, Single binary, single config file, runs on a single node up to 100k events/sec, Apache 2.0 license
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) Adoption hinges on the belief that platform engineers prioritize operational simplicity (single binary, no ZooKeeper) over existing industry standards like Kafka. [platform_engineer, single_binary_deployment]
|
||||
- (0.85) The system assumes that achieving wire-compatibility with the existing Kafka producer protocol is sufficient to mitigate the risk of platform engineers rejecting a new, fundamentally different architecture. [kafka_wire_compatibility, eventstream]
|
||||
- (0.80) The design assumes that the technical complexity of implementing first-class backpressure signals (REQ-001) will not introduce new operational overhead that negates the simplicity benefits over Kafka. [req_001, single_binary_deployment]
|
||||
- (0.75) The system assumes that platform engineers are willing to adopt a new streaming solution primarily because of its backpressure semantics (REQ-002), rather than due to other core business needs or performance metrics. [platform_engineer, req_002]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Achieving the required 100k events/sec throughput while maintaining single-node, single-binary deployment complexity may lead to significant performance bottlenecks when handling complex backpressure logic. [single_binary_deployment]
|
||||
- (0.85 · high) The requirement for wire-compatibility with the Kafka producer protocol may conflict with implementing a fundamentally new, first-class backpressure signal without introducing complex compatibility layers. [kafka_wire_compatibility, producer, eventstream]
|
||||
- (0.95 · medium) The model does not specify how the system will manage state or offset tracking for consumers, which is critical for reliable event processing and consumer-driven flow control. [consumer, eventstream]
|
||||
- (0.75 · medium) The system's reliance on the 'Platform Engineer' actor as the target user does not account for potential operational complexity in integrating a new, non-standard protocol into existing enterprise infrastructure. [platform_engineer, a1]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-001 states that producers must receive backpressure signals as a first-class part of the protocol, but the 'producer' block only lists 'backpressure_signal_input' as a property and lacks any association to enforce this protocol requirement. [req_001, producer]
|
||||
- (0.85) Requirement REQ-002 states the system must default to consumer-driven flow control, but this requirement is satisfied by 'eventstream' without any associated block or association defining how that flow control mechanism operates. [req_002, eventstream]
|
||||
- (0.75) The 'producer' block includes a property named 'backpressure_signal_input', but no association or requirement links this input to the actual backpressure signal mechanism defined in the system. [producer, eventstream]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It correctly identified the core functional blocks (Producer/Consumer) and the target actor (Platform Engineer).
|
||||
- **model.accuracy:** 3/5 — The use of generalization for Producer/Consumer is acceptable but weak; composition between Producer and EventMessage is questionable.
|
||||
- **model.parsimony:** 4/5 — The model is focused on the core components without adding unnecessary or fabricated entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were captured perfectly as dedicated constraint blocks.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., operational simplicity vs Kafka) and tied directly to the seed's core tension points.
|
||||
- **risk.detection:** 5/5 — Risks are substantive failure modes (performance bottlenecks, protocol conflicts) and domain-specific to event streaming infrastructure.
|
||||
- **voice.character:** 4/5 — The findings are highly technical, skeptical, and concrete, maintaining a professional PM voice.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a very strong run that excels in identifying deep technical risks and constraints. Minor improvements could be made to the model's structural associations.
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLJ1Rjim3BtxAuZq421eWYpjYgueYdR3XkwoTiFk1ctHjb1Pz2GvWLSKs4Vi2_SbevoaJRq4knWoIR_t-4ZvsWTqeQrDuX-qRT1X3Hdc3wNZrgeFRDZ1KV6M2bhiLFWA5Q-qBQ50usadanlq7XuJWFV_eAW2YtdCi5FaDj7i7D-TBoHwunolDi4pUgjEb0G_iGrVi2OOtP9PKj0vZfAd7JxBWGxGpUaaFpr0zvHKMX5GtOGEQgepSZv9rk97znJWkqFh3GPs8q0F3OKqF8VWyj9tFb1zTGL7Y_mKpx9U0xPASohIsf3wu4ZqNO0Sn2Z9BeNnS9Pg3fej6ibXeBHcHKOACcP3Q4NnZhgxbQKNOHpVH1FceaqaHIaL-bHoEm0GkTgoDEJJbGvLgYXd5MUudsGBBy2PuVnX0xpa78t-YL5igZ3WRXCPgourQCiMInfadx4_sq2kSHHw3BXhWpZIi-bNcPJxp81_u9FWasthnxnrchqEtR7IqLplZnk3NT4Q6yzLQqjoJz44iO1aTYhDgCAbbcdkIDnglBDJHKkTasWgo96LKo6bZTDBpBkrD-jpciMbGzU9q9ojGA7qqQjzRlcMHKK0bD80eI9GkjZ2PXHMH1R6cyxMLqWNikhnTGne5OotozT7j0Ms6QEBu-krz6cv4OBOfGqRdU-LaxoiotHwDTpg2sWz-LYpFziuZZliDpiSE1cKJARJ0pEyW9A4LGOc-4FVPhCzPDWqHWkNiEnhxd_V98Q4KO7JPHLax9F9D_hHQaSrsU0datNeuzVfVFvcCe4_ltx3_QOzG9c9u2mnCzIxBxR88WrQ0oxwKOgAiNzrBVflubM0xLMOxT0jDdGtnl3AGpyD6I17bXvmIBDsK0eo5jnuTMEPTf5vbaYJwrQJQtd8Z_ml)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Pet (block), Interpretation Output (block)
|
||||
- Associations: pet_owner→pet_translator:uses, pet→pet_translator:provides_input_to, pet_translator→interpretation_output:generates
|
||||
- Constraints: On-device inference for privacy, Honest about the difference between 'emotion classification' and 'translation' in onboarding
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The system assumes that providing an 'emotional state' and 'suggested behavior' (via interpretation_output) will be perceived by the pet owner as a useful, believable translation.
|
||||
- (0.85) It is assumed that the user base (pet owners) will actively provide continuous audio and video input streams to the system for analysis.
|
||||
- (0.75) The model assumes that 'emotional state' classification alone, when combined with suggested behavior, is sufficient to fulfill the desired outcome of helping owners bond with their pet.
|
||||
- (0.60) The design implicitly assumes that the technical feasibility of combining audio and video inputs into a single emotional_state_model exists and can be maintained on-device (privacy_boundary).
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The system's core value proposition, 'translation,' is fundamentally misaligned with the technical output of 'emotional state classification', risking user disappointment and skepticism if not managed by the onboarding process. [interpretation_output, transparency_policy]
|
||||
- (0.90 · high) Relying on 'on-device inference' for processing both audio and video streams introduces significant technical risks regarding required computational power, potentially leading to poor performance or high battery drain. [privacy_boundary, audio_input_stream, video_input_stream]
|
||||
- (0.85 · medium) The model does not specify how the 'suggested_behavior' within the interpretation output is generated or validated, creating a risk that the advice given to owners will be unhelpful or even harmful. [interpretation_output, req_002]
|
||||
- (0.75 · medium) The system assumes that 'pet' provides both 'vocalization_data' and 'body_language_data', but the model lacks mechanisms to handle missing or corrupted input streams, which could cause a complete failure of the interpretation process. [pet, a2]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association 'a2' from 'pet' to 'pet_translator' is structurally redundant because the system already defines inputs via properties on 'pet_translator' (audio_input_stream, video_input_stream).
|
||||
- (0.90) The requirement REQ-002 states the system allows owners to use the interpretation output to bond with their pet, but no block or association explicitly models this user action or outcome.
|
||||
- (0.75) The 'interpretation_output' block has properties ('emotional_state', 'suggested_behavior') but no explicit associations connecting it to the user experience or subsequent actions.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured core actors (Owner) and entities (Pet), but missed modeling the necessary 'Veterinarian' or 'Behaviorist' expert consultation role.
|
||||
- **model.accuracy:** 5/5 — The associations are logically sound; using 'uses' for the owner and 'provides_input_to' for the pet is appropriate.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean, though the inclusion of a separate 'interpretation_output' block might be slightly over-engineered.
|
||||
- **constraint.capture:** 5/5 — Both explicit constraints (privacy/transparency) and implicit requirements are captured accurately in the system definition.
|
||||
- **asm.detection:** 4/5 — The assumptions are highly specific (e.g., technical feasibility of on-device ML), moving beyond generic platitudes.
|
||||
- **risk.detection:** 5/5 — Identifies critical failure modes related to the core value proposition and technical constraints with high domain specificity.
|
||||
- **voice.character:** 4/5 — The findings are highly structured, skeptical, and concrete, though some prose could be slightly more economical.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run that correctly identifies the core feasibility risks inherent in the seed's premise. The minor deductions relate to missing expert roles and slight structural redundancy.
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.95
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNBRXj74BplLpJaYqsO0aFPXYCOWXF1iNEG3J_kYzcPtkM0kpFhUP2X7G7v27zXlYGr-n0faHlu8W7TCzNLDLNBrpu85s9JPtwjJIkSQAWGSbqv6uswjhLrzAXypYKl3qxubL1sgqr5fQWz7tHaBRodxndHxqSegXJb8dMiK-o6Ql5I_FfoYUflpjdjK7p1pzM5Gl4FQy9xqJ2TlUDwmq5BSPRT7impDl34k0LVo6SJuswpbLPCtBHXHmqt1JkVPJtviuzHSw1tDZfFd-MALQpPdP7mz3LrybNgv7xiqAjNVkS3DrTNz6WfduaNnOZqfr5werKKDLrZ4wUB66mFfUzQkTotW2Ka_WCgBIL5pKO9bmif6O2NL5XRip3ejGwFuXohlBGo-Zneo7H97b2cme83XJUcqeOv5Ub6654Dg_2-dZTz_S7mmAB9lVwMK4rC4krHhmTYz9dTHalk8E_O-hu8lAAsSZtWYLRdOTScE0RUzyhQgkPylFMqqT9PRykGLqwqgrjSSInKldKwyB42Trnk6Bx0Tf_uQsGVEYfDNyjTNtl01DQ1atGhJF1xoPAajhKUYjxNaaX59uAsvb29SVmdS9nt7Ebjjn7TYIECeWynrDQkwV7Dk1wzJUizWGC2Lprkn_bWsOCMgCCVO29GxFZl5u_OpB5lMGPAbBzPmwHq2L-mWLp4VxKr5Wtr3jE6zH3ORjnu8HyHfWUzjoJWpWQ497L29hstEY09WIAUct2rrAxfAPSmSbWneGA65mjIOkVfSH5rhIWuq7uoP407ZG1qfRo56QqwHIER2ilVy_dfJ6OdxJrz_7Imv_Eh8rDF79q1-e7fiea7FJ-_clYGN28ZU316cEt_tJuAshsBHm_xamDFtdiOf7itJhxVJqxBQgwm13bThG8YDfkb_6h73IELirbVUlDnlbZyCflHl__ye2-mLF-bfIPPdusFZZkdTHvqN6jHmEiL8zSY4BuJEBPCvWowYZPwKipjV6lTklUn9x2cBVFw_63WScAWuSfYOIGTKGgGY1Lr7zykJB3pHdkT21Ix8Mmf9w8Ex0nURyEZdDrvVpZoOc8alkzHzZjwAo4sba7PbkJ5BY4PIm7nLFELbICya25jBt3s6d_m-_uV)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours Scheduler (system), Individual Contributor (actor), Engineering Manager (actor), Calendar Service (block), Meeting Request (block)
|
||||
- Associations: individual_contributor→quiet_hours_scheduler:, engineering_manager→quiet_hours_scheduler:, quiet_hours_scheduler→calendar_service:, quiet_hours_scheduler→meeting_request:
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The system assumes that users will adopt a negotiation mechanism (REQ-002) that is perceived as less socially costly than outright declining an ad-hoc meeting (REQ-003). [req-002, req-003]
|
||||
- (0.85) The system assumes that the target users (Individual Contributors and Engineering Managers) are willing to grant read-write calendar access (read-write-integration) for a tool whose primary function is scheduling negotiation. [individual_contributor, engineering_manager, read-write-integration]
|
||||
- (0.75) The system assumes that the ability to propose alternative times (req-002) is sufficient to guarantee a user reliably gets at least 2 contiguous deep-work blocks per week (REQ-001). [quiet_hours_scheduler, req-001, req-002]
|
||||
- (0.70) The system assumes that the proposed negotiation mechanism will not be circumvented by users manually overriding or ignoring the protected blocks, even after the initial trust period (initial-trust-period). [quiet_hours_scheduler, initial-trust-period]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's ability to negotiate protected blocks invisibly (req-002) is undermined if the `meeting_request` block cannot model the *reason* for the conflict, limiting its negotiation intelligence. [meeting_request]
|
||||
- (0.85 · high) The `calendar_service` block only specifies read-write access (read-write-integration), but the core function requires complex, real-time conflict resolution and proposal generation, which may exceed standard API capabilities. [calendar_service, read-write-integration]
|
||||
- (0.95 · medium) The model lacks a mechanism to track or quantify 'political capital' (req-003), making it impossible for the system to reliably measure success against the desired outcome. [quiet_hours_scheduler, req-003]
|
||||
- (0.75 · medium) The `individual_contributor` and `engineering_manager` actors are modeled as general types, but the system's core logic depends on differentiating their specific needs (e.g., IC vs EM meeting patterns), which is not captured in their properties. [individual_contributor, engineering_manager]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The 'individual_contributor' and 'engineering_manager' actors are generalized to the 'quiet_hours_scheduler' system, but no associations define how these specific user types interact with or are used by the core system logic. [individual_contributor, engineering_manager, a1, a2]
|
||||
- (0.75) The 'calendar_service' block is defined with properties like 'api_type' and 'read_write_access', but the system lacks any explicit associations or requirements linking these properties to the core functionality of the scheduler. [calendar_service, a3]
|
||||
- (0.80) The 'meeting_request' block is associated with the scheduler via composition (a4), but no requirement or association dictates how this block is generated, modified, or consumed by the system. [meeting_request, a4]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (IC, EM) and services (Calendar), but missed key concepts like 'ad-hoc meeting' or 'political capital'.
|
||||
- **model.accuracy:** 3/5 — Generalization associations are used correctly, but the composition of Meeting Request is weak without defining its lifecycle.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean and focused on core components; it avoids unnecessary filler blocks or entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly translated into the model's constraint section with appropriate scope.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., negotiation mechanism perception) and tied directly to desired outcomes/constraints.
|
||||
- **risk.detection:** 5/5 — Risks identified are substantive failure modes (API limits, political capital tracking) and domain-specific.
|
||||
- **voice.character:** 4/5 — The findings maintain a skeptical, analytical tone, focusing on gaps in logic rather than just missing features.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run; the detection of risks related to 'political capital' and API limitations shows deep domain understanding.
|
||||
|
||||
---
|
||||
362
phase-0/reports/2026-04-28-2129-run.md
Normal file
362
phase-0/reports/2026-04-28-2129-run.md
Normal file
@@ -0,0 +1,362 @@
|
||||
# Phase 0 corpus run · 2026-04-28T19:29:51.026Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 8 · **OK:** 7 · **Failed:** 1
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 4/3/3/3 | 1/4/3 | 0.90 | 4 | 3 | 4 | 5 | 3 | 5 | 4 | **4.00** | **3** |
|
||||
| 02-habit-coach | medium | 6/5/3/3 | 4/5/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 03-redline-ai | medium | 5/4/4/2 | 4/4/4 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 04-skillswap | medium | ❌ | — | — | — | — | — | — | — | — | — | — | — |
|
||||
| 05-cyclist-thing | failure-prone | 2/1/0/0 | 1/3/1 | 0.20 | 2 | 3 | 4 | 1 | 2 | 5 | 4 | **3.00** | **1** |
|
||||
| 06-eventstream | hard | 5/5/3/2 | 3/3/3 | 0.95 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 07-pet-translator | failure-prone | 5/5/2/2 | 4/4/3 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 08-quiet-hours | medium | 6/5/3/2 | 4/4/3 | 0.95 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
|
||||
**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed
|
||||
|
||||
**Passing:** 6 / 7
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/bLN1JXin4BtlLrXmYvee1AWeGWYAGBrGKU2-chLd4pTUUpls9arHf7v4lx1Vqh6pIJQ5LEe5mSpiUyzldisL3q2XgKpcfzhMG5290kHqJAwnwiOPH-AmFCKIHvq9Fm7bvjgEHGd6OwSZ3NWlNZ8XFhv2KIMKmzXnf93QQd467yv6NBqcSlEs-1vFrR7YufspuJDKA0u-ePbXq18EicM7pxg0U-Y6U2nFzj0jiubMAB2gmq9KM1L8FijMuW-kIVlWWi430Lx0-YznSU4NFc1rUIaEH_84tXU9MLiVT6Y2TXPCNaG9G0jnBWhd38Bb4UFcEHizHPzNZdctlUMQz2aq2cr8b7xrEnE238wOB_98rv3772qvOwhOF-T9uYrm4oGehBJC3SxGS0TjKxsm5AGnWMeWu79BTfEWvewcufuxfC4aOhcfvrLRPn65SNBQYl1DyHLbo84GklnAbwMMZGcBNGNQ1dAgYSTOl0Cr0ojHTPGyJxITnY0zS0mjadX2xzd992UqpRnkctzfMOtcgY68xZCdK_8gqr9OFhiN-vEumBIHLkvoi-hxnWTXMJ09ruIw2GBOXAesoB7opgIjYk2u76r49MfoXS4gwQluwpnzdVFiB2P4Eau30TCdaPjpFgAlkON2iAzMBaHQZJWQZBnuE1r6YfhG8p5AoN59-EriNlF0VkYxsyU7Qy6J475___pLMMDxDexQ3FrkKiKwgOcgHAeXNuEyGPQjGzdlNtRknFd6QP_FTPXasrOSV2jGvmAOVl43_GxcgsN718rXp7T6s-oLrOF1hWoeQwDvDhYiQzi_ndQF_5-3Uuwr_IGpE4Qh1Edn970SUxr7_DPemdXlVQ-tAjr-wG-7HxrUMjhpX1ENtfUKAtvPfJOwG0nYQnDXVsL9kltHxkwgrqe67V3H7d1EyIm-W12VflxgQTfiGAmtWDzhHcGliOjvl0Tpt4J4CDdg3MuEf5PHPmwh8dYKpNUjbH0hYdsHONFT_21ZwrO6PbVyW_yx_G40)
|
||||
- **Post-validate:** stripped 4 refs, dropped 2 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Student (actor), Coursework Material (block), Thinking Partner Session (block)
|
||||
- Associations: student→aristotle:interacts_with, aristotle→coursework_material:analyzes, student→thinking_partner:engages_in
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.75) The system's refusal to solve problems is sufficient to prevent academic dishonesty accusations from faculty. [must_never_solve]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Maintaining FERPA tenancy for coursework material requires robust, verifiable data isolation mechanisms that are not detailed in the model's element definitions. [coursework_material, ferpa_tenancy]
|
||||
- (0.80 · medium) The latency constraint (latency_p50) is applied only to 'aristotle', but the actual interaction flow involves multiple blocks and associations (student -> thinking_partner), potentially introducing cumulative latency. [aristotle, thinking_partner, latency_p50]
|
||||
- (0.90 · high) The model lacks a mechanism to enforce the 'Must never output a complete solution' constraint (must_never_solve) during the actual generation of responses in the thinking_partner session. [aristotle, thinking_partner, must_never_solve]
|
||||
- (0.70 · medium) The association 'a3' (student -> thinking_partner) does not specify how the interaction history is managed or retrieved, which is critical for maintaining a coherent low-stakes conversation. [thinking_partner, a3]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) Constraint `must_never_solve` applies to the system `aristotle`, but the requirement `req_003` describes a behavior (guiding without direct solutions) that is not inherently contradictory to, nor fully satisfied by, this constraint. [must_never_solve, req_003]
|
||||
- (0.90) The requirement `req_002` states the system must provide a low-stakes thinking partner experience, but no block is explicitly associated with providing this 'thinking partner' functionality. [req_002]
|
||||
- (0.75) The association `a3` links the student to the block `thinking_partner`, but no requirement explicitly mandates that the system must facilitate this specific interaction. [a3, req_001]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It captured the core actors (student) and system components (thinking partner), but missed key institutional roles like faculty or curriculum structure.
|
||||
- **model.accuracy:** 3/5 — The associations are generally correct, but 'analyzes' is too strong a verb for coursework material; it should be more about content access.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids excessive filler blocks, keeping the scope tight to the core problem domain.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (FERPA, latency, no solutions) are correctly captured and applied to relevant system elements.
|
||||
- **asm.detection:** 3/5 — The assumption about academic dishonesty is specific and tied to a constraint, but it remains slightly speculative rather than purely falsifiable.
|
||||
- **risk.detection:** 5/5 — It identified high-severity, domain-specific risks related to data isolation (FERPA) and system mechanics (latency/constraint enforcement).
|
||||
- **voice.character:** 4/5 — The findings are highly technical and critical, maintaining a skeptical, expert tone without resorting to excessive prose.
|
||||
- **Average:** 4.00 · **Min:** 3
|
||||
|
||||
> This is a strong run; the risk detection was excellent, particularly regarding system implementation details. Minor refinement of association verbs would elevate it further.
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRjGm47tFLupAIrcHghG5bQggW7AJA51eUOuczYInrh63xUoge4fy15_8bp1siZHjT_kosXsFvyoSSyRxqWTqeMzrvkVATEYmXHB5l7QsD_BKQklWKVMCAjgRPFW6fLqgKqE5sjFaH6Zq7dvb0A_lLP4LLhlnn3f9ReoMX_ZYS8-Zhvopop7ud9x9VSd1TzQ4pzWIR7qWlQ2W16vbrnCyOmDjWDkbVN6m0UuwQvGaeBOBLz1IMvBpMJOql_K1InNWrA9ejW0zDF5t8U9lE3xsLpvGUt82Z_R40Jul4xh4W0NMjQCQ0-CSGMcj9ZHyvgYpBZ1PXJA1N8M29kVNAzHpT6H20kpILyP2mSKOAaAmJZM5elSKo_lWkEBDxTD6QGdRX_dJWyUfXaY1_oLMUHV24HcuLCpf6suwvVeKB087YpW9Nokr5VCH6RkEOR6cGjlw1fjRYXTynxqM7Rc2y_ZKz97DcyxEL9lOD9MgUuT1MPD0TOmpgvFu7Lo9IbyL0YTLkHcjxP9a1FFhUB1cmLeEvIwPuu6CIRG87Bs7nF8jbEzHgv_3tQc2dRDibAXWIw91etnxIy4L-AVU1rXQD-SkhQbzj9ZwSW5Jqj58U6zjhGdUOSi3mfaoyoVHoD4fEheZDDIclfNrHPMoD2SnMMt7tPLAAxRjyJ6FnVCZsohDSSSCISasjbJIQAcSRmHdjIO9h1YSVZa_VmNRtoyU3tszEV-MVuJjjzz73zckmwAMdHkmjKB3xMq6_cp1Mw5GGqKOUaSUAjw6kP8U_lx-mqC2KyRRM5fqqiUfeUjBhKHyKgAI2TVOOYXJ_2-p1ZCRroJFJswjw16qQDYjll0KecgUfSLiM8ZxoGj5Ir-C3i1iUXYJx-t64QpsdFojx5cUhp7Q4IWZT2yvUTBlElEpcIdux47DTtRkjizxe1NF6cosGQu7xwoX-i7yJ5DDR1MdwYRmQinct-X7hnorF9gVpORGsw_vxkxJsIo9VjaG38_cu4P-0ECmp1yWy8lD4j6JvDPyUC3Pd5CUsRvX6VTdv2ywDOcvCm7Rsm36gxk8QyZBlUAG7MJZYW5L5OcW5cI8pMkhKUlqh0qA-Id4_WQ8z2ADpNdWsYnZQdAmuTggJE50PFQIF_Zlzny0)
|
||||
- **Notes:** Habit Coach is the SoI; Parent and Child are actors; Screen Time Data and Limit Configuration are first-class blocks.
|
||||
- **Lint warnings:** 1
|
||||
- Requirement REQ-002 satisfied by unknown block: reporting_interface
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (8-14) (actor), Screen Time Data (block), Limit Configuration (block), Conversation Tool (block)
|
||||
- Associations: parent→habit_coach:manages_settings_via, child→habit_coach:views_data_via, habit_coach→screen_time_data:aggregates, habit_coach→conversation_tool:includes, parent→limit_configuration:sets
|
||||
- Constraints: Must work alongside iOS Screen Time and Google Family Link, not replace them, Child data must be parent-controlled per COPPA (US) and GDPR-K (EU), No social features for kids — no leaderboards, no public sharing
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The system assumes that parents will value a data-grounded conversation (REQ-001) more than the immediate satisfaction of enforcing limits, which is necessary for the 'Conversation Tool' to be adopted.
|
||||
- (0.85) It is assumed that aggregating raw screen time data (a3: aggregates) will automatically translate into actionable insights and a measurable improvement in parent-child communication, which is required for the 'Reporting Interface' to fulfill REQ-002.
|
||||
- (0.80) The model assumes that parents are willing to use a system (habit_coach) that requires them to actively manage settings and interpret data, rather than simply relying on existing OS-level controls (ios_familylink_compatibility).
|
||||
- (0.75) The design assumes the 'Conversation Tool' can effectively shift the parent's role from enforcer to facilitator without requiring significant behavioral change or education for the parent.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on aggregating 'Screen Time Data' from external sources (iOS Screen Time, Google Family Link) without detailing the necessary API integration points or data synchronization mechanism, creating a high risk of technical failure. [a3, screen_time_data, ios_familylink_compatibility]
|
||||
- (0.85 · high) The 'Conversation Tool' must synthesize raw data into actionable insights and conversational prompts, but the model does not specify how it handles conflicting or ambiguous usage patterns, risking a failure to meet the core requirement of facilitating a routine conversation. [conversation_tool, req_001]
|
||||
- (0.95 · high) The 'Limit Configuration' block is associated with the parent setting limits, but there is no defined mechanism for how the system enforces or reports on limit bypasses, which contradicts the core problem statement that existing tools are easily bypassed. [limit_configuration, a5, req_003]
|
||||
- (0.75 · medium) The 'Parent' actor is responsible for managing settings and setting limits, but the model fails to define a clear workflow or UI element for how parental oversight (e.g., reviewing data logs) translates into actionable changes within the system. [parent, a1, reporting_interface]
|
||||
- (0.90 · medium) The 'Child (8-14)' actor views data via the system, but the model does not specify if this viewing is read-only or interactive, which impacts the child's ability to participate in a collaborative conversation. [child, a2, conversation_tool]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.95) The requirement REQ-002, which demands visibility into the effectiveness of screen time limits, is satisfied by 'screen_time_data' and 'reporting_interface', but no association links these elements to the parent or child actors who need this visibility. [req_002, screen_time_data, reporting_interface]
|
||||
- (0.85) The 'conversation_tool' is required to facilitate a data-grounded conversation (REQ-001), but its properties ('data_visualization', 'prompting_mechanism') are not explicitly linked to the source of the data, which is 'screen_time_data'. [req_001, conversation_tool, screen_time_data]
|
||||
- (0.80) The 'limit_configuration' block is associated with the parent setting limits (a5), but there is no requirement or association specifying how these configured limits are enforced or consumed by the system. [limit_configuration, a5]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (Parent, Child) and key system components (Data, Limits, Conversation Tool).
|
||||
- **model.accuracy:** 4/5 — Associations are generally correct; using 'composition' for includes is acceptable but slightly imprecise.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and avoids adding unnecessary or fabricated blocks/entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints are correctly captured and applied to the system block.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., behavioral shift) and tied directly to core requirements/tools.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (API integration, bypasses), and address failure modes.
|
||||
- **voice.character:** 4/5 — The findings are structured, analytical, and skeptical, maintaining a professional PM tone.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run. The model excels at identifying technical risks and regulatory constraints while keeping the scope tight.
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRXCn47tVhnYL5uX8LT8MGOKgIhXLAY2W4e_Mh7TsO-hBOdiR8gZ4H_25V0bZxpPTIDENA9xBEUEPCzxd8QAFdT55k52sHOy6IfGNZNUThUPEEm_twaEgQJQA20kit5BP1chKWKOUgJ44-540lDX0gMgizvB7-OhyO2sVuDCdCxQUUE-MW_4n7LRx5HjVEnlVeo7OUKlwagAIk5DSZVYiYxI5Re_svS4MkgjYeIe2CcrSWI5JaWz5qHU_yuaghIp1oUaEO03V7mKgUFOih48aStmCzsRo01-NcLlPI8t7g9mLhSPOEs_W24Bqt2BsCsuNI6XLUlGhWL8Iynn1wPmcj5pDm7n63Meu9pIPMQUZY7naPfJHUIPEX9xpE9ziPtuqPCbpcEIx1z8FltPMEIyOpqc5aQvkuEVSJSyuyDB9pf2DcKKELb4DLYOhjPCN0rdo9-lcXR1jDK-aL5fn6s-vpzoP5kqAvZaUple63ApP9VxjpN_KB4vReQ-6g32kuP97bXgUotVwDlhhGNxkceP26iy_0mrhy_-qlOEOTu6seJ6j9mpEydzHOb2ZYgxM-ddNXRZM1C5InGMyKzAxuEe8NrYEkGp3OM8T9bAOMBAJY-4xCIgoCrUqES3hcIGCiCIzW1mFYILC9PkTOQDn5Ua0zrklBXNBYLAc96_3Wro3TI9dYEiCaM7IPc-lugIhLCny6bmDYNq5GwT6FSsrbRnatrld8vQQCkNDKM024ZtGThhF7-OmW_DLIt1wcdMZ5V8beFREG8KhSBQ7PiNFHEH0ePGO1Tw2NOntw-TqEjhoOhJmq-dnvdvi1jmcvM9phoOZdfj65Lk5iBjxF7vzZY0l70KHNN77vEvAszBzkrAsTVMkdDHUiXLurImYRyrayec-TSfJwaEOJ7hJgu_JlRr7amdy-VKRNYiVuhHD8nzG0mzs_GIinz0lJgy3ulLYqHjbkqXXTuGw6r3F5mHl-XSIvlqBsUTxkbIqv3T-9Dd0SkMtcvMbBCTVCx9aINOfOBVWI_LNAvxp3tyt_m80)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Company Clause Library (block), Redline Suggestion (block)
|
||||
- Associations: legal_team→redline_ai:, redline_ai→contract_document:, redline_ai→company_clause_library:, contract_document→redline_suggestion:
|
||||
- Constraints: Must integrate with Microsoft Word, Contract content must never train cross-customer models (privilege concerns), Audit trail of every change suggestion must be exportable, SOC 2 Type II compliance from day one
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) Legal teams will accept the speed of redlining (req_001) even if the suggestions require significant human review time (req_002). [redline_ai, req_001, req_002]
|
||||
- (0.85) The AI's suggestions will be perceived as helpful and trustworthy enough to replace the mechanical work currently done by junior associates (problem). [redline_ai, legal_team]
|
||||
- (0.95) The system's ability to suggest changes must be directly and reliably linked to the company's specific clause library (a3). [redline_ai, company_clause_library, a3]
|
||||
- (0.80) The system must provide a clear mechanism for the legal team to reject or modify AI suggestions while maintaining an auditable record (exportable_audit_trail). [redline_ai, exportable_audit_trail, legal_team]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Achieving the 'minutes' latency target for redlining (req_001) may be impossible when constrained by required integration with Microsoft Word and complex, multi-step document parsing. [redline_ai, must_integrate_with_word, req_001]
|
||||
- (0.85 · medium) The requirement for an exportable audit trail (exportable_audit_trail) adds significant complexity and overhead to the core redlining logic, potentially impacting performance targets. [redline_ai, exportable_audit_trail, req_001]
|
||||
- (0.95 · high) The 'no cross-customer training' constraint (no_cross_customer_training) necessitates a complex, isolated inference architecture that could introduce significant technical overhead and latency. [redline_ai, no_cross_customer_training]
|
||||
- (0.90 · medium) The model does not specify how the 'Company Clause Library' (company_clause_library) is queried or prioritized against existing contract content, risking irrelevant suggestions. [redline_ai, a3, company_clause_library]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-002 states that the General Counsel must review AI suggestions, but no association or block explicitly models this required human interaction between the legal team and the redline suggestion. [req_002, legal_team, redline_suggestion]
|
||||
- (0.85) The association a4 (composition) links the contract document to redline suggestion, implying that suggestions are inherent parts of the document, which contradicts the idea that suggestions are generated by the system and reviewed separately. [a4, contract_document, redline_suggestion]
|
||||
- (0.75) The block 'legal_team' is defined as an actor but has no associations connecting it to the core process blocks (e.g., redline_ai or contract_document), making its role structurally isolated. [legal_team, a1]
|
||||
- (0.80) The block 'company_clause_library' is defined but has no associations or requirements linking it to the core process (redline_ai), suggesting it may be unused. [company_clause_library]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured core entities (Legal Team, Contract, Library) but missed the critical 'Microsoft Word' integration platform as a modeled dependency.
|
||||
- **model.accuracy:** 3/5 — Using generalization for Legal Team to Redline AI is weak; composition between document and suggestion is likely incorrect.
|
||||
- **model.parsimony:** 4/5 — The model is generally focused on the core domain elements without adding unnecessary filler blocks or actors.
|
||||
- **constraint.capture:** 5/5 — All four explicit constraints were correctly captured and applied to the system block in the model structure.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., trust/speed trade-offs) and tied directly to process requirements and user roles.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (Word integration, data isolation), and tie back to explicit constraints.
|
||||
- **voice.character:** 4/5 — The findings are structured, skeptical, and concrete, though the prose could be slightly more economical in places.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run; the risk detection was excellent. Minor structural fixes to the model's associations would elevate it further.
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
**Status:** ❌ failed
|
||||
|
||||
```
|
||||
Model JSON failed schema validation:
|
||||
constraints.2.confidence: Number must be less than or equal to 1
|
||||
|
||||
Raw response (first 600 chars): {"blocks":[{"id":"skillswap_platform","label":"SkillSwap Platform","kind":"system","confidence":0.95,"properties":[{"name":"exchange_history","type":{"kind":"string"}},{"name":"user_reputation_score","type":{"kind":"number"}}]},{"id":"remote_worker","label":"Remote Worker","kind":"actor","confidence":0.95,"properties":[{"name":"marketable_skills","type":{"kind":"string"}},{"name":"desired_skill","type":{"kind":"string"}}]},{"id":"service_offering","label":"Service Offering","kind":"block","confidence":0.9,"properties":[{"name":"skill_type","type":{"kind":"string"}},{"name":"scope_hours","type"
|
||||
```
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.20
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TP5BQm913CVlyod2UXPaVM111bj1FFNKUydEPDp5UKWoLfRYT--8UzYsUFtb_qZ8LZ9olWIlvDJ5Cp86QD2SZfmkqUwIJmmJjoH7zKWXBTfqxU8H77gXqSHu585l1V3sByKwTBFx9B4b7cYpnfTrNUWhSxeES4LBExS5xbFCxnW8gWFvBygTmKhTHdqnPNfIDwEvMJofkwcsimGKphc7GA4X5gKUorUxtlXECdoqvS8AKC0yo6U-4zXif9TCGMkOr6Q1g-QFKyQc8aUJ4sjT2WOqdUhVcKfjAThoXXy0)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclists (actor)
|
||||
- Associations: cyclist→cyclist_thing:
|
||||
- Constraints: _none_
|
||||
- Requirements: _none_
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The product will be accepted by the target user group, cyclists. [cyclist, cyclist_thing]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The model lacks specific functional requirements, making it impossible to define a measurable success metric for the 'Cyclist Thing' system. [cyclist_thing]
|
||||
- (0.90 · medium) The target user ('Cyclists') is too broad and lacks defining characteristics (e.g., amateur vs. professional, type of cycling) to scope the problem effectively. [cyclist]
|
||||
- (0.85 · high) The system definition ('Cyclist Thing') is merely a placeholder and does not articulate what specific problem it solves for the 'Cyclists' actor. [cyclist_thing, a1]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association 'a1' connects the actor 'cyclist' to the system 'cyclist_thing', but no specific interaction or flow is defined. [a1, cyclist, cyclist_thing]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 2/5 — The model only identifies the actor and system but misses core concepts like bike types or cycling activities.
|
||||
- **model.accuracy:** 3/5 — The structure is technically correct (actor to system association), but it lacks any meaningful relationships or composition details.
|
||||
- **model.parsimony:** 4/5 — Given the vague seed, keeping the model minimal and only including core elements is appropriate and highly parsimonious.
|
||||
- **constraint.capture:** 1/5 — No constraints or requirements were surfaced at all, failing to capture any meaningful non-functional boundaries.
|
||||
- **asm.detection:** 2/5 — The assumption is generic ('will be accepted') and fails to push back on the vagueness of the target user/problem space.
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, correctly identifying the vague nature of both the system and the target user group.
|
||||
- **voice.character:** 4/5 — The findings use clear, skeptical language that questions the scope rather than accepting it, maintaining a good critical voice.
|
||||
- **Average:** 3.00 · **Min:** 1
|
||||
|
||||
> The model correctly identifies the vagueness of the seed in its risk detection but fails to push back on the assumption and lacks any structural depth.
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.95
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRjim37ttLn3jMr1dRdf1LmH1kwx3W673jXROeq5BT294bZn9JX1i1VOH-y9zoUXRubxI5mCYfNCEoIFvoWTqeIfqv9VAbEYmW0p5SkviPUIDrTR1ONv6EKq6E_m2fLqhCuSSjQT1HcZq7dv50E-UeSWSywJEM2V9TT7i0jzUJ3XwxPnTTy5pEfCdaeCVh0bVi20u-4XwHK49F8WU1dp61jf3bz29EDr3zn0jb2IWeWmRAAZ8oFaeQiKVtAx8XBlW28i3G0zKhtspXkdKRtoWOZQ3muaunVEiOI-T3LPOdQuOI5a3by07k46SgzjPElA-SfHwDJUeEPrPgmaDA-bOlpehAq6keIox1VEX2DOnNKsZqSmhd5EQAIFhzk-ZiPadjy9CQSK5li1sOuslYevDT8idR2n2C1x3fmwvpPTWghfJdCgrNQTyB3YkMM3v2bFJJNYF0NVDJ2KlcItJLYnxDjneYt9OKb05iJ-BSaUzXRstfHBmcGBA7ZhKeRJeGi_WNL21HyEoAnOqACPHO4rrgYIdx432mzOJFvIZMDYYnA0oJR1MOG6VC5_YRbQz2I1dfnLgpashhU0nfx1f52xh0-cQiT8Ug-tRT5hF83XK9ZI2jtNUiGxcuu6Zsno1RvUyErTy-vIc8t2LyS14s6UDPMzN9JS4ZfDasHduZIVHg6XtfIraAgdKTbFKGzgluhf4iI2OZ1FGIf3nr21X4qudQP9suPSmefsRVyVny3f5E-SzpMpFnF5iO9hTYIVnuIKTCTPZCqHxEpyUpnvTy4ikgjIAV1fijBTJhvvweIklxeyqpSb8S6g-2Ep7qUWx_QnOQatbHwCsTFijJfBZqGZ-_VaB_MFXeQXyO0SBKYjwz0P0-zHuK2REaE7hUR5Zd0znEzNwIe3D8V1aU--E1sIJZkoUy-shrx99oh7IeRPNFurOEkOtpRi0tRimZhYejhJeYZ_ya_aF)
|
||||
- **Notes:** EventStream is the SoI; Producer and Consumer are actors interacting with the core EventStream system.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Producer (actor), Consumer (actor), Event Data (block), Topic Metadata (block)
|
||||
- Associations: producer→eventstream:, consumer→eventstream:, producer→event_data:, consumer→event_data:, eventstream→topic_metadata:
|
||||
- Constraints: Wire-compatible with Kafka producer protocol for migration safety, Single binary, single config file, runs on a single node up to 100k events/sec, Apache 2.0 license
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) Adoption relies on the assumption that platform engineers prioritize operational simplicity (single binary, no ZooKeeper) over maintaining compatibility with existing Kafka tooling. [single_binary_deployment, kafka_wire_compatibility]
|
||||
- (0.85) The system assumes that the ability to observe backpressure in-band (req_001) is a sufficient differentiator for platform engineers who are already familiar with Kafka's operational model. [producer, eventstream, req_001]
|
||||
- (0.75) The design assumes that the target user base (platform engineers) will accept a system that fundamentally changes core streaming semantics, even if it maintains wire compatibility. [eventstream, kafka_wire_compatibility]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Maintaining wire compatibility with the Kafka producer protocol while enforcing a first-class, in-band backpressure signal (req_001) introduces significant complexity to the eventstream's core protocol design. [eventstream, kafka_wire_compatibility, producer, req_001]
|
||||
- (0.85 · medium) The constraint of being a single binary running on a single node up to 100k events/sec (single_binary_deployment) limits the system's ability to scale horizontally, contradicting the implied need for high throughput in modern event-driven infrastructure. [eventstream, single_binary_deployment, producer]
|
||||
- (0.95 · medium) The model does not specify how the system will handle topic metadata changes (topic_metadata) or schema evolution, which are critical operational concerns for platform engineers migrating from Kafka. [eventstream, topic_metadata]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-001 states that producers must receive backpressure signals in-band, but the `producer` block has no property dedicated to modeling this signal. [req_001, producer]
|
||||
- (0.85) The `eventstream` system block lists `backpressure_signal` as a property, but this signal is fundamentally related to the interaction between the producer and consumer, not an intrinsic property of the stream itself. [eventstream]
|
||||
- (0.75) The `producer` block lists `backpressure_observability`, but this property is a direct consequence of the system's core goal (REQ-001) and should be modeled as an explicit requirement or constraint rather than a general property. [producer, req_001]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It captured the core actors (Producer/Consumer) and system (EventStream), but missed key operational concepts like Schema Registry or Topic Management.
|
||||
- **model.accuracy:** 3/5 — The use of generalization for Producer/Consumer is acceptable, but associating them directly to EventData via simple associations lacks specificity regarding the flow control mechanism.
|
||||
- **model.parsimony:** 4/5 — The model is focused and avoids unnecessary filler blocks; it only includes necessary operational components like Topic Metadata.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly transferred to the system block, and the requirements captured the core functional goals effectively.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., operational simplicity vs compatibility) and tied directly to conflicting constraints/goals, making them actionable.
|
||||
- **risk.detection:** 5/5 — The risks identified are substantive failure modes (complexity of wire-compatibility + new semantics) and domain-specific, exceeding generic warnings.
|
||||
- **voice.character:** 4/5 — The findings are highly structured, concrete, and skeptical, maintaining a professional, critical tone without excessive hedging or fluff.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNRRjms47tNLmoS1mEBjU7OJf0QXf4BN3X0qAHjq5TYH8wqX2cEIbBQAe61V4I_i5_I4NVblTY6yo98mvavPmuFgRSnOKXTuufuQtsB0HieKT_MWJjlFh3Z02-gLrJHwLP6NA3XfVKrLEWYRQreXp72zmBW_OCkfiBgP5pXO2Yie-KR_ENDgKJVXS3BTV0rlJ9dHeA_ia-_OKDmS4EkfsGr7XHtMtYU4pq1Tq9d-lm9kBjYOGq1DMqQeA6cf12BOaN-u0ib-1hGHuU9mm5WX9QIIlSXkBoCGqpKN5t1YrDzZg_Bp458TKarRD3PP2d21P3lck_O6SlpNX3vBgTL5CXhKkHhwqd97gHkp8ufYAZ2Rel9vwMd3GaUlmGVjV0G-14sanjQck0CrtEDQOIYVreADcCvwiazWV4NQw7y3PDbVm_LRmK5iNIiRzU8UI1bCE6c8K0LwEzEi0ObIjKZ7M4ZokvYlMSpm2T9wB2cUx1IeigjetjeMRW7Q2r7KhSqj6nzUaoywuP7wkZWJz6NuADF5Df0QJCcJIbv2qZPdPGz6csm3OP1hOicjLk7GzMvkUw2vIxEiQu3nMXxccjnB_cy4V9QoHZZHaJDOKUSYV1dVsIejves3e5At2R0FUeXCsQlLZbgatDvATrbUXJwcUkz13ViAIR0ahi4QK5WR3LLbPIMH1uEftboZQtaY8sVXu3UmE5a-Z6I2InoIsLib7pRP3C-W2ysLZqwkje_FHV0uahHEnPxArFoZlKkH03M8Yb5bK_OKrMx7lgficSCS04r-HLL9H9EJJQJFLiVI9DO8QgAo8mNQ_7O3XuVFo8IjgsJIqGbBXwA_bn5uKYe6mYsNYJntspsXnnE6wWHGyRPR1Mw_ltevEJbR0R__VWNlee_LjSQD9suHkGRJnwWM4NQzbWwogRe8edXOMzMhaP2ahl0td8uta8utKBOgyf8v7FlqQ0hQHD3oOAqj6anXcqOUnyNCj5ghkAjFEIlzJy0)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Pet Vocalization (block), Pet Body Language (block), Emotional State Interpretation (block)
|
||||
- Associations: pet_owner→pet_translator:operates, pet_translator→pet_vocalization:processes_input, pet_translator→pet_body_language:processes_input, pet_translator→emotional_state_interpretation:generates_output, pet_owner→emotional_state_interpretation:receives_feedback
|
||||
- Constraints: On-device inference for privacy, Honest about the difference between 'emotion classification' and 'translation'
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) Pet owners will accept an output that is a 'believable' interpretation, even if the underlying science is merely emotion classification. [req_001, req_002, honest_disclaimer]
|
||||
- (0.90) The system assumes that the owner's desire to 'bond with their pet' (REQ-002) is a direct and reliable outcome of receiving an emotional state interpretation. [req_002, a5]
|
||||
- (0.80) The system assumes that the pet owner will provide sufficient and varied input (vocalization + body language) for the inference engine to generate a useful interpretation. [pet_owner, a2, a3]
|
||||
- (0.75) The system assumes that the technical feasibility of inferring emotional state from multiple modalities (audio and video) is achievable while maintaining on-device inference. [on_device_inference, pet_translator]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Achieving a 'believable and useful interpretation' (req_001) of pet behavior requires integrating diverse, real-world behavioral data that is not captured by the current limited input modalities (pet_vocalization, pet_body_language). [req_001, emotional_state_interpretation, pet_vocalization, pet_body_language]
|
||||
- (0.85 · high) The constraint of 'On-device inference for privacy' (on_device_inference) severely limits the complexity and size of the underlying ML models, potentially making it impossible to achieve the required accuracy for complex multimodal inputs. [on_device_inference, pet_translator, pet_vocalization, pet_body_language]
|
||||
- (0.95 · medium) The system relies on the user perceiving 'emotion classification' as a 'translation' (req_002), creating a significant market risk if the output is perceived as merely scientific data rather than magical insight. [honest_disclaimer, pet_translator, req_002]
|
||||
- (0.75 · medium) The current model lacks a mechanism to handle the variability of pet species and individual temperament, risking failure when applied outside of highly controlled test environments. [pet_owner, pet_type]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.95) Requirement REQ-002 claims that the system enables bonding, but no block or association explicitly models this outcome. [req_002, pet_translator]
|
||||
- (0.80) The `emotional_state_interpretation` block is defined as the output of the system but has no associated requirement that mandates its specific properties (`primary_emotion`, `confidence_score`). [emotional_state_interpretation, req_001]
|
||||
- (0.75) The system requires processing two distinct input modalities (`pet_vocalization` and `pet_body_language`) but only has one associated requirement (REQ-001) that addresses the overall interpretation. [a2, a3, req_001]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (Owner) and inputs (Vocalization/Body Language), but missed key concepts like 'pet' itself or the concept of 'behavior'.
|
||||
- **model.accuracy:** 5/5 — The associations are logically sound; processes_input and generates_output correctly map the data flow from input to interpretation.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean, only adding a few necessary blocks (e.g., Interpretation) without excessive filler entities.
|
||||
- **constraint.capture:** 5/5 — Both explicit constraints (privacy/honesty) are captured and correctly applied to the system block in the model structure.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., owner's desire to bond is reliable), moving beyond generic platitudes.
|
||||
- **risk.detection:** 5/5 — Risks are substantive failure modes tied directly to the seed's core tension: feasibility vs. user perception.
|
||||
- **voice.character:** 4/5 — The findings are highly structured and analytical, maintaining a skeptical, professional tone appropriate for this domain.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a strong run that correctly identifies the high-stakes technical and market feasibility risks inherent in the seed.
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.95
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLNRZjGu47tFLvI65sZHiqqF8BQ5Hi0i2pmCkupsFQh4bRJLZXriPtfdqKXy15_8bt2Sf2yxquruwKvEkUeSbqyvhq9a7xl6P66bRSkU6oguNDNUTLPTEECyFQYUIINplHLXoSgjjQsfOXDaBr8Q3e6-P4HlxbLH5LUp575UYH_HuWN__c8EzBNtRZs2p-MPEbC0_tGsVkH6wEIzc6k9kkIJx7QFpxeeH-XcSbO-FK9tcosr4fAcZJVKI5E83rasY3_vr6c9zDvrFfmG1_gStlDbUi_3JOZIqCkNmyFvEJsObq_vUR59_c2LljQgOqCN4EzrqKNd-pfw6yhBNGYrkCG_IgLzb6p4AlOvbwMWu88AvuomXUQHugsjjHNngV-NRBcMeRxiyBmPy5tnJVB5M9_-uCXzsfPH0K52OLov6jNmDMl3XJOQVTf9eVIq4fK7usBI669YtKcy59mLv5t9vqv2x7cQ0SlzWDrXIgZsAAawpr4xYwgsImU3OEjTwmA2KSCBz-d-zh116KNHcrIofsitMDxJtA5J8cs-TdwLOm1Yb_OWeFk2RkbhUToY8s2uJIllwRdTEkLAM4tNNiE3xvohZT2slMmL_TL5WpxHmqjTUXTS5UcTvtRv25Q8KWzrUx5rdvrlZs8l3k5m2tRC07lrkovtQAENq6AVb5hpdxD2IbULUB6m3yc_hT6bZkO6b6CRCR4zvoOXtrGuJlJH4SEh3GILrBSn7UvQHmn5f2x0Q4Yjj6y6oHLi7PT2GA3mR4QARm8zB3fj54KFsO_6YT1HOnnwA6zXNQSEoSYEZCvqUdveEXT9KiWEpSFFCmv4KiB_9sI1bhCAl_NdVZJdhjiNgVjfee-dt7NiWWRL8VkPFKvFZ-o8MvozX3owxDvH_qhImSFvbSJCI0sTv7MzZ9YWoUHg6Ev6R0oJoG2z_JITpPvC9lJzwpVw1sOPIpM3gOtc0guL6pellPbwczM2-uOZuHh2idcoNTHrvp31QQIdQQG7X-86mFgroEfqZt1-X73i_LwH_ghPM5pRilFzXHjSYQQ05qbmOEjmDUAh18HR7TcSmdPgQ43s2Z_ujFu0)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours (system), Individual Contributor (actor), Engineering Manager (actor), Calendar Data (block), Meeting Request (block), Protected Block (block)
|
||||
- Associations: individual_contributor→quiet_hours_system:uses, engineering_manager→quiet_hours_system:uses, quiet_hours_system→calendar_data:reads/writes, quiet_hours_system→meeting_request:processes, quiet_hours_system→protected_block:manages
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that negotiating protected blocks invisibly (proposing alternative meeting times) is sufficient to eliminate the social cost of saying no, which may require more than just calendar manipulation. [req_002, initial_trust_period]
|
||||
- (0.85) The system assumes that the user's primary blocker to deep work is fragmented calendar time, and not other factors like context switching or lack of motivation. [quiet_hours_system, individual_contributor, engineering_manager]
|
||||
- (0.90) The system assumes that the user will adopt a tool that actively intervenes in their existing social communication patterns (meeting requests) rather than reverting to manual coordination or simply ignoring the tool. [quiet_hours_system, a4]
|
||||
- (0.75) The system assumes that calendar integration alone is sufficient to manage deep work time, and does not account for the potential friction of conflicting data sources (e.g., Google vs. Outlook) or API limitations. [google_calendar_integration, timezone_respect]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's ability to negotiate protected blocks invisibly (req_002) is highly dependent on the success of 'processes' association a4, which must handle conflicts with existing booked slots in calendar_data. [a4]
|
||||
- (0.85 · medium) The model does not specify how the system will handle a scenario where both the user and the meeting organizer are external to the defined actors (individual_contributor, engineering_manager), potentially blocking necessary meetings. [quiet_hours_system, a3]
|
||||
- (0.75 · medium) The constraint 'initial_trust_period' (no automatic blocking for 30 days) creates a dependency on user behavior, which may fail if the system cannot reliably prove its value before the trust period ends. [initial_trust_period, req_001]
|
||||
- (0.80 · high) The system relies on 'reads/writes' association a3 to calendar_data, but the model lacks detail on how it will reconcile conflicting write operations (e.g., two different systems attempting to book the same time slot). [a3, calendar_data]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The association `a1` (Individual Contributor uses Quiet Hours System) and `a2` (Engineering Manager uses Quiet Hours System) imply that only the actors themselves interact with the system, neglecting the necessary interaction between the actors' calendars and the system. [individual_contributor, engineering_manager, quiet_hours_system]
|
||||
- (0.75) The `calendar_data` block contains properties for 'availability' and 'booked_slots', but no requirement explicitly mandates the system must read or write both of these pieces of information. [calendar_data]
|
||||
- (0.85) The `meeting_request` block has a property for 'proposed_time', but the system's core function of negotiating alternative times (as stated in the initial hypothesis) suggests it must also handle proposed *alternative* times, which is not captured. [meeting_request]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model correctly identifies the core actors (IC, EM) and system components (Calendar Data, Protected Block). It misses explicit mention of 'Meeting Organizer' as a necessary external actor.
|
||||
- **model.accuracy:** 4/5 — Associations are generally correct, but the `a1`/`a2` associations should link to calendar data or meeting requests, not just the system itself. The model is mostly sound.
|
||||
- **model.parsimony:** 5/5 — The blocks and associations are highly focused on the problem domain without adding unnecessary filler entities or concepts.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly captured as dedicated constraint blocks with appropriate application scope.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., social cost, context switching) and tied to the core mechanism of negotiation, avoiding generic platitudes.
|
||||
- **risk.detection:** 5/5 — Risks are substantive failure modes (API conflicts, external actors, trust period dependency) directly related to the domain's complexity.
|
||||
- **voice.character:** 4/5 — The findings are highly structured and technical, maintaining a skeptical, analytical tone suitable for deep PM review.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
---
|
||||
397
phase-0/reports/2026-04-28-2144-run.md
Normal file
397
phase-0/reports/2026-04-28-2144-run.md
Normal file
@@ -0,0 +1,397 @@
|
||||
# Phase 0 corpus run · 2026-04-28T19:44:44.425Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 8 · **OK:** 8 · **Failed:** 0
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 4/4/3/2 | 4/4/3 | 0.85 | 5 | 4 | 5 | 5 | 4 | 5 | 4 | **4.57** | **4** |
|
||||
| 02-habit-coach | medium | 6/5/3/3 | 4/4/3 | 0.90 | 5 | 4 | 5 | 5 | 4 | 5 | 4 | **4.57** | **4** |
|
||||
| 03-redline-ai | medium | 6/5/4/2 | 4/4/3 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 04-skillswap | medium | 5/5/3/3 | 3/4/3 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 05-cyclist-thing | failure-prone | 2/0/0/0 | 1/3/1 | 0.20 | 3 | 4 | 5 | 5 | 4 | 5 | 4 | **4.29** | **3** |
|
||||
| 06-eventstream | hard | 4/3/3/2 | 3/3/3 | 0.90 | 4 | 3 | 5 | 5 | 4 | 5 | 4 | **4.29** | **3** |
|
||||
| 07-pet-translator | failure-prone | 5/4/2/2 | 4/4/2 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 08-quiet-hours | medium | 5/5/3/3 | 4/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
|
||||
**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed
|
||||
|
||||
**Passing:** 8 / 8
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.85
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VPLHRziu3CVV_Ic8xYr8WYnRXrrG5EiL6tOFE_HwksU3jcXR5rdo92f1C1JOXxXFU9_aADl9ELtRbo2XAF97yY_cGs3q75kJXQss7NfieS1oMtiNhRfrndbuLLrIHUk9HsXGkRssDLHe0arEIeCXmFSCuDUVegWAgrKwSLwH7wt5U_pb_LgiDzwx_MXyHvVgZHBZ9sVvTsm9BZwJsH7h4Y-oXqa-wvYUIRUYD-NRPz8zP8rM1DHsV82Mse9yoB81_kB6wy2E3Lq01i3ZBxYw2eV0r5vVmwjr-HRV5NrcRGDhZgoTHPCN2G7z0JPGE6S8hRXmewqWrxb0ioMVjqwoRu1iRB__wKgFKXbyYnHI576StpfRKiSH3RJOTSdq-T0vRYZe09mSRVqWTOpCVrcfiVQe8ifX3nxFJ4AF9Jily0aQIrJKwZ8tj2CZA86zH9IJqaKVQE_yDXTjS0n7pixJmb39qHDuscdQppiN-6GhdGtEQ8MfXddZsZw8a0WnevGpmRqz9O4lGkSrcfvvahmTxG9U65TkH_0GYxybOOwUS8gjT5NfCXe-d4hggRLbxrHCfH_cyeDyIqfBx3cgNQ9I4woluvpWRfWJ_9OkID-aefxlyI0VmHctJqtRKiZ9xhHtjYNBPrAO33bNCk56JeLJ12WOHuulCJ1OASE3Yzn51fJMj9qXKRiqkHSRi1Dp6Z0fwBmh3BKzP2kty_vsFjpEJrUkhjAKsADqecS-bNvFePCZ0YEziULXa0-yNgu3t5skKZ8HG22_8wYapZxJw9jtul1yw4yVx-zkHBCMKzn_V_mp6VDOfJHd9nezUqfmV4fzgefyX_ano1F9iaTwNook9szt0teSPSZtcfkdl9yItsQKUiZ-ZvLyVnB2HbPXgOtc5EwHyzE1PMGbXI3UBmnikJolGdQ1qI8lTjcq-I-udJVk1SVCK4rMWTTrmwBKsUoUlaNjAMaup6Q3wUCVYzNgzMpMp_5h8tBiTs8lkaaBqbDO34z1tc6DTHy5Kkkdqp-sOpa9lducU1tJoWuSLU99WfI5DCGUufvwgs9Q2ZAZX96-e1Or4U-9B8phIXQxrEhIqY1e4j4oa-g77cGVv4F-5Fy3)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Undergraduates (actor), Coursework Material (block), Thinking Partner Interaction (block)
|
||||
- Associations: undergraduates→aristotle:interacts_with, undergraduates→coursework_material:reviews, aristotle→thinking_partner:facilitates, aristotle→coursework_material:processes
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The product assumes that simply refusing to give answers (as per 'must_never_output_solution') is sufficient to convince students to use the tool.
|
||||
- (0.85) It assumes that the current market saturation of answer-giving tools represents a significant and exploitable weakness in the existing educational technology landscape.
|
||||
- (0.75) The model implicitly assumes that institutional adoption will be achieved by focusing solely on FERPA compliance, without addressing other potential administrative or pedagogical hurdles.
|
||||
- (0.90) It assumes that students' disengagement is primarily caused by the lack of a low-stakes thinking partner, rather than other factors like course structure or workload.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The constraint 'FERPA tenancy — coursework never leaves institutional boundary' implies a high dependency on specific, potentially limited institutional IT infrastructure for data handling and processing. [aristotle, ferpa_tenancy]
|
||||
- (0.80 · high) The requirement to maintain 'Response latency under 1.2s P50' while processing complex, potentially large volumes of 'coursework material' may require significant optimization or specialized hardware not accounted for in the model. [aristotle, latency_p50, a4]
|
||||
- (0.95 · medium) The model lacks a defined mechanism for handling the transition from 'reviewing' coursework material (association a2) to generating meaningful, low-stakes interaction via the 'thinking_partner' block. [a2, thinking_partner]
|
||||
- (0.90 · high) The constraint 'Must never output a complete solution to a graded problem' is difficult to enforce purely through system prompts and may require complex, verifiable guardrails on the underlying LLM generation process. [must_never_output_solution, aristotle]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association `a3` (from `aristotle` to `thinking_partner`) implies that the system facilitates the interaction, but no requirement explicitly mandates or describes this facilitation role. [a3, req_001]
|
||||
- (0.90) The `thinking_partner` block has properties like `low_stakes_environment` and `questioning_depth`, but no requirement or association links these specific properties to any functional need. [thinking_partner]
|
||||
- (0.80) The `coursework_material` block is associated with the system via `a4` (processes), but no requirement states that the system must process coursework material. [a4, req_001]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 5/5 — The model successfully identified the core actors (undergraduates) and system components (thinking partner, coursework material).
|
||||
- **model.accuracy:** 4/5 — Most associations are correct, but 'processes' for coursework feels slightly too strong compared to the desired interaction/review.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and does not introduce unnecessary or fabricated entities beyond the core scope.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were captured perfectly in the model's constraint list.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., market saturation) and tied to core hypotheses rather than generic platitudes.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific (FERPA, latency), and directly reference model elements/constraints.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, but the prose occasionally drifts into overly academic phrasing rather than direct questioning.
|
||||
- **Average:** 4.57 · **Min:** 4
|
||||
|
||||
> This is a very strong run that accurately captures the constraints and risks inherent in the problem space. Minor refinement of association labels would make it perfect.
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLR1RXit4BtlLn3EfHKanxMJ87K38wvT9qNRs8tJCp5Bpgu8SJbRaYj53Gpa8_A5_P8EkUju5Kb6BuB4uSnxV7mprEiGmSUkiKLO6DU2XqQKe1Qrfyxf2xBanPFgELPuFDeHvg1fPLmjAh01Hn5b8GJngH3YvwqgkeBgA4N8Q_J3QlaIVdnvpAldtjDgM7o1p_M9viKhSl4TD2WEtg9TOZGA3egx4PwZY7lWZl14FTi3TrVCZKQ1JHlNei6cH1-AeYT_y1PA4yK5WPeV20XYddvBbNwBLw_2EaHipix4asFr35wK6Lr310brxR66o3YdeYIo28vZ7bloaSMInaNq5IWSnU_kKM_0euiPiCrV6GiK5sEe1A7cnchPUggCJGL2z5oJ0pM1bG5ZXb3GCbjh-5WxC2vIZGoHgts3q0NqV4mN65nQNABb6kYwvfEbrRH1RRfcEZVr_8ut1-KHdOocGTa5g4UKli9TujBmOQzlnNVc-lRfkTEUZFu-4z0v9YamYTAIMWma74IpH3akdyJTK1HQ8q4f366JfaVGafnTJrVUH9mwSdZtGEUlJFEIgsK6cNLVV9C2j2th6B4clnuBhJlFvCZ9nhWkOk0Oex9rHWAJMxALyhO793LQtGKOgJLgZ9HkW4_Ern0KUHm3VKMwlhavV_hcykRzxBUCn3vdyuDV3v2j1rdhrYzaEGGODTrkzC167A2p8x9zHTE5A4eSt3VZhT6JjQW5E2qKEEwr5EVczxouCd4kujnudJAYOJtI4dLHuCVM6iKzr9S2AmPN7JuenQRubTkXxYNT94yXjyeGUvJ_7udqYln2W2LNXzJLx3Lnc_qZFh2AcVuRefhRvWeQOzVYT-CMFzqVYVlJffxCIdDgrJcLS25rqOZnEnAtf0oVvWeXTfuFV4Dypfwp8nboL5RtqHrqFynPlpn0UhaJVkZQD204cvldL4dWTPYAjYkvTXQRFJ4LF5GKDIsFmDGVmeDRfFdR87j6XKIpU6YfsUniREzJmMDZoT84wM4bXnnPaI_6isqhwtwWSHvhA1tvXaNv1yDMshR3J_bfK6aGvNHjWkf2I57Wwup5CE9IxhZMgLWQNF5-2FEiX5mQAFfXjRqtprEUO44cv_LRT_d-yF3RJ6vnouwLaVPdR4hnFn9sYJ3As-VgHzdjiDMZ-mkBDRBVFO_fo8QRJDxZtvtncAP9c4pwfL_-d1qT_J2PY7y_Vzdo9EUpgloimj3gqpmVPlthofqVkgP9nAaQNW6HXbMlbs1JYHNYWhlCOxhCmn7ayHx81wiaqEWxbOwgDnJbqHDN2MomJXvAsHZPKGDy2v4zuS8OzMGFwj0KRBbWXZUJBuXugF4_Wwf2bJhDyIkJJeeVJKWlxCQHhMbCP2nMlDUzUCqV_6_gFm00)
|
||||
- **Notes:** Habit Coach is the SoI; Parent and Child are actors; Usage Data and Conversation Data are first-class blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (actor), Device OS (iOS/Android) (block), Usage Data (block), Conversation Data (block)
|
||||
- Associations: device_os→usage_data:provides_raw_metrics_for, habit_coach→usage_data:aggregates_and_normalizes, habit_coach→conversation_data:facilitates_discussion_about, parent→habit_coach:views_dashboard_via, child→habit_coach:interacts_with_via
|
||||
- Constraints: COPPA/GDPR-K Data Boundary, OS Integration Boundary, No Social Features Policy
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that parents will find a shared, data-grounded summary (req_001) more motivating and less confrontational than existing methods of monitoring screen time. [req_001, parent]
|
||||
- (0.85) The model assumes that the raw metrics provided by device OS (device_os) are sufficient to generate a structured conversation (req_002), rather than requiring qualitative input or behavioral modeling. [a1, req_002, usage_data]
|
||||
- (0.80) The system assumes that the child will actively engage with and provide data for conversation (child) via habit_coach, despite having a defined user consent level. [a5, child, habit_coach]
|
||||
- (0.75) The system assumes that the parent will trust and utilize the visibility into existing limits (req_003) to improve their relationship with the child, rather than simply using it as a punitive measure. [req_003, parent]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's ability to provide visibility into the effectiveness of existing screen time limits (req_003) is constrained by relying on `device_os` and its limited `api_access_level`, potentially preventing detection of bypasses. [device_os, req_003]
|
||||
- (0.85 · medium) The model does not specify how `usage_data` is linked to the child's specific activity or device, creating a risk that aggregated data will be too coarse for meaningful conversation (req_002). [usage_data, child, a1]
|
||||
- (0.75 · medium) The `parent` actor's ability to view the dashboard (a4) and set goals is not explicitly tied to the child’s consent level, creating a potential conflict in data ownership or visibility. [parent, child, a4]
|
||||
- (0.80 · medium) The `conversation_data` block and its association (a3) do not define the mechanism for generating a 'shared, data-grounded summary' required by req_001, risking the conversation becoming purely subjective. [conversation_data, a3, req_002]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-003 claims the system must provide visibility into the effectiveness of existing screen time limits, but no block is associated with tracking or reporting on the *effectiveness* (i.e., whether a limit was successfully enforced or bypassed) beyond just raw usage data. [req_003, habit_coach, device_os]
|
||||
- (0.85) The 'parent' actor has a property 'goal_setting_capability', but no association or requirement suggests that this capability is utilized by the system (e.g., setting goals for usage limits). [parent]
|
||||
- (0.75) The 'device_os' block has properties related to native data and API access but is only linked via association a1 (providing raw metrics) and requirement REQ-003, suggesting it might be underutilized if its full capabilities are not needed. [device_os]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 5/5 — The model successfully identified all major actors (Parent, Child) and necessary system components (OS, Usage Data, Conversation Data).
|
||||
- **model.accuracy:** 4/5 — Most associations are correct, but the relationship between Parent/Child and Habit Coach could be better modeled as a composition or generalization.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused; no unnecessary filler blocks or overly complex entities were introduced.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (COPPA/GDPR-K, OS Integration, No Social Features) were accurately captured and applied to relevant components.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific, focusing on behavioral dynamics (trust, motivation) rather than generic platitudes. Strong effort here.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific, and directly challenge the feasibility of core requirements.
|
||||
- **voice.character:** 4/5 — The findings are structured, skeptical, and concrete, though some prose could be slightly more economical in tone.
|
||||
- **Average:** 4.57 · **Min:** 4
|
||||
|
||||
> This is a very strong run. The model excels at identifying deep technical risks and regulatory constraints while maintaining structural clarity.
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLRRRXit47tVhnWuBwaG6QuI1waG677L9Z7WD6XY88y4bvnTCUPbItAjge61VaI-C5-I8NSbqvRa5mCxbtCEvoQ_2P7xs1jTXMjbE-wvWPgBwzQxtig5qyx3a-OK6vmL4M79fLifsqB3TS328pGF0RvL0B_leCY6DoV9uxn4FrhhL_otLpEoddllLgFn9PxAvvACRvsDVt63SFGUzGr69VXHTLlmMHVn0Dq9FXSl3j3TLaib4T1qSGq6JOq-LDKW_kWJIgqimld54V00VlXaNC7hrs4T8fgpCtWo4o_uopfpAnknzJmgPrcdUMoSDp2743sLYFo4smTaMjMU-pKBwZyaj-qJRN8xqsd5hK16tOXz82UIzrk7LXB2iqr0l2NXeyXBRBc6A-GcYzJfaqNw995SHET9OzBcdSODNeiMFOK9AbD0_UnhRvNpZF2SK3nY0R-Wmdl2WJ-Sw0tQc5d4Q6Loj19PhPsu7icIFrbttyuxUe7WjTAAAZw7sf4kRaj2qt6xXaMEXykXLYDhTh7xPNn8dPsXfBwXfb9BqfZk8Tnq-NFVjXXIx-vrEspD3uX61qecbjosM39-xIcZeNVclB8ApZOqSBd5mlcY_LizvxrK4IvTcsNmzCMqQn-mHsM8d9kkHAU7KnV53cdMnoH6hdHHcTljf7yWlTlfHLYfk8GFIdWNN1FX2orE5cCeZ6t3s8hi98lUH82Sx5dRxkXifY6bWqLg1-HuI0HXAiZi31cDaqZwddPUtIWQP4oP0hqDlsHwwrZEO9iCbc5INm-h6CgPF1fS0ubzFLO4YjvcRJLTWdyxvoElDMRAkqysT680Eapt-UC2Pd2rxX0kBegDXiOx0vAlmTa1cdPjnY85CgLO4RW7kogsUZezAo_HF3KzCBeqpEEDmbLLE5FixghEeVDEOA2ytU2z6pOdnWPz6jjxAPFfTD-Up64uC76Nu6wS4oPjW0mqKDM-mJe-Vl3GV3hehT5LZqp2Osa7klbOog4kFPQJsaKN6xngbv7sRpBvXF_qocCgVvXC1jEVVqzFJdwTJE37_z_XhV8XJhiqGYDgs8xaffYIDXICiZtLzRZ0c8qOLqk4TyDbXyLmsGUCOJ3ePwmO-41WTBouJ4l1-xXqwIJSaUTn3O9unVAuejSCRwhUq1_wB-4d)
|
||||
- **Post-validate:** stripped 1 refs, dropped 0 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Company Clause Library (block), Redline Suggestion (block), Audit Log (block)
|
||||
- Associations: legal_team→redline_ai:uses_for_review, redline_ai→contract_document:processes, redline_ai→company_clause_library:references, redline_ai→redline_suggestion:generates, redline_ai→audit_log:records_in
|
||||
- Constraints: Must integrate with Microsoft Word, Contract content must never train cross-customer models (privilege concerns), Audit trail of every change suggestion must be exportable, SOC 2 Type II compliance from day one
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.85) The system assumes that the General Counsel (GC) will accept AI suggestions as a primary review mechanism, rather than viewing them as merely an efficiency tool. [req_002, legal_team]
|
||||
- (0.75) The product assumes that the legal team's primary pain point is time spent on mechanical redlining, and not a deeper issue of knowledge transfer or process standardization. [legal_team]
|
||||
- (0.80) The system assumes that the 'Company Clause Library' is sufficiently comprehensive and accurate to cover all necessary legal variations encountered in real-world contracts. [company_clause_library, a3]
|
||||
- (0.65) The product assumes that the technical feasibility of generating accurate redlines is independent of the complexity and quality of the input contract document. [a2, contract_document]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system's ability to generate suggestions in minutes (req_001) may be compromised by the necessary overhead of maintaining a comprehensive audit trail (exportable_audit_trail) and ensuring SOC 2 Type II compliance, potentially leading to unacceptable latency. [redline_ai, req_001, exportable_audit_trail, soc2_type_ii_compliance]
|
||||
- (0.85 · high) The requirement to integrate with Microsoft Word (must_integrate_word) introduces significant technical risk regarding reliable data extraction and manipulation of proprietary document formats, which could fail during the core redlining process. [redline_ai, must_integrate_word]
|
||||
- (0.95 · high) The constraint against cross-customer training (no_cross_customer_training) necessitates complex, resource-intensive on-device or private cloud inference architectures, potentially impacting the feasibility of achieving rapid redlining suggestions. [redline_ai, no_cross_customer_training]
|
||||
- (0.90 · medium) The model does not specify how the 'Company Clause Library' (company_clause_library) will handle versioning conflicts or ambiguity when multiple clauses apply to a single section of a contract, risking incorrect redline suggestions. [redline_ai, company_clause_library]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The requirement 'Audit trail of every change suggestion must be exportable' (exportable_audit_trail) is not fully satisfied because the association 'records_in' (a5) only links the redline generation process to the audit log, but does not explicitly link the *suggestion* itself to the record. [exportable_audit_trail, redline_ai, a5, redline_suggestion]
|
||||
- (0.80) The requirement 'Contract content must never train cross-customer models (privilege concerns)' (no_cross_customer_training) is a data governance constraint that applies to the system's *behavior* when processing the 'contract_document', but currently only links to the system itself. [no_cross_customer_training, redline_ai, a2]
|
||||
- (0.75) The requirement 'Must integrate with Microsoft Word' (must_integrate_word) is a technical constraint that applies to the system, but does not specify how the output or interaction flow will occur within the 'contract_document' block. [must_integrate_word, redline_ai, a2]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core entities (Legal Team, Contract, Clause Library) and necessary outputs (Suggestions, Audit Log). It missed explicit mention of 'Microsoft Word' as a required integration point.
|
||||
- **model.accuracy:** 5/5 — The structural relationships are sound; compositions (suggestions from AI) and associations (processing/recording) are correctly used to model the domain flow.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean but includes a few generic blocks like 'redline_suggestion' that could potentially be merged or simplified into the main process flow.
|
||||
- **constraint.capture:** 5/5 — All four explicit constraints were accurately captured and applied to the system block, demonstrating strong adherence to the seed input.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (GC acceptance, Clause Library completeness) and tied directly to core elements, avoiding generic platitudes.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific failure modes (latency vs. compliance overhead, Word integration risk).
|
||||
- **voice.character:** 4/5 — The findings are concrete and technical, maintaining a skeptical tone by focusing on implementation gaps rather than just stating problems.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run that excels in capturing domain-specific risks and constraints. Minor improvements could involve linking the Word integration constraint more explicitly to the process flow.
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLR1Zjis4BthAmQRcn4RszqaIDrWaJHDqQ15a7GBz2YCo95Dc29L3cN7M2pGZ-WNzaio56LRYUMY5zjuGyrxd9atyckE66BNs88tnhKOi84ArMOLVEVqMszzW2VrSwhfPdI2rwZzphWLr6YPHX5baHaU2e0VpxBe6klh5F51KnZGwYL-__96q3SX-Dq0lg3d-bOB-BDtyGCs15U_aDrID0glYiSHd_EHBj1TqwrwTe7kiLWRJK1D6_VGK5DHuAB8ug_kDyRQ-nssyD5Yh7rehW0PEC4iSDaECBnwnNkErDpTmPCRzGnVLBqM-gpMw5PKil8jbTOq9i8IN9Tu9DxWvvA9sNXNwYvWb1-d-ED1nw_Exopf5S6VFcmez2es1x3SzQ1eG1Lz40aFVMe18rPMo9DUISinIASa5gZjOa-MX0MQeEol3X-3husbqwtBDYD2LbclDWEPhsiAf3CJZwaSaPu8d7XITNyp70NDDAckDWFVi0JwgpD7cZ4BoV46euJaU-ewxuP6m7skTazrwau6lw5JlcajvIf9jReadoJnGqjE2-_JuG3ffzgmL9JqOrA9lVIHXeE83nuQxsWFCQ1ZQTP1YFDb7oX70H6Zl9CRed5nKFJ7ck0uSat7CKszE4yPQqGX0RfzJo5MtX-l1nN57P63ZcMu5wUYlDVaef6vtr8mjJZgACWCaN8ScLBqHlhGfyro3XKv9lWgDMmDIg_TXlHx1s925ZzOb6yAMwC8kbOEiLcvhXt9_2aNbq5XsvA6A6KaRL1cNKPYQsZNQnvwKERO1RrllQkjKL4UP6-x_bYll2BetPdp7ueypya4p4G_W9Gqh2WjHWOKvuWVrgPApoN5nPafv_ExgNsnJ5NZ_pX-DDeIL22HmCL4cgdZ3JfSNSmzTjSIMi_nOk8pUon7eyR5vD0k5XUkAwspXhYClfWUhlJacUIldZjlyFzXAoojnAyGp6eTPG9ciz_p6cb41Szc6NhtQNvz_Tri1l_-_S-4pRXJIefGTzQAmJ2gDSITJ_id5KYclqAM8H59sG4fTx9W5hSOSToCEF8h8pECJ1mb98hdqS-JyDCQaZKBc7FsZOITYMiZi2CGWs_6FBSNUCXn9sDR-RW6UN7AUzIqA8L8QskWFgmD6GuJpPRiVb68a5p7uhLyoB-4Bm00)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: SkillSwap Platform (system), Knowledge Worker (actor), Skill Profile (block), Swap Listing (block), Exchange Session (block)
|
||||
- Associations: knowledge_worker→skillswap_platform:uses, knowledge_worker→skill_profile:creates, skillswap_platform→skill_profile:manages, knowledge_worker→swap_listing:posts, skillswap_platform→exchange_session:facilitates
|
||||
- Constraints: No money transaction, Identity verification, Disputes capped to mediator review
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The platform assumes that users will actively use and maintain their 'reputation_score' (knowledge_worker) to facilitate high-trust matching, rather than relying solely on the initial LinkedIn verification.
|
||||
- (0.85) It is assumed that a two-sided marketplace structure will inherently solve the problem of casual exchange by providing sufficient visibility for non-cash skill trades (skillswap_platform).
|
||||
- (0.75) The model assumes that all necessary skills and exchanges can be adequately captured within discrete, bounded 'Swap Listings' (swap_listing) rather than requiring more fluid or open-ended collaborations.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The model does not define how 'complementary skills' are determined for matching, risking failure to connect users who genuinely need each other. [req_001]
|
||||
- (0.95 · high) The 'reputation_score' property on the knowledge worker is not linked to a mechanism for portability or calculation, undermining the core hypothesis of high-trust matching. [knowledge_worker]
|
||||
- (0.85 · medium) The system relies on 'dispute_mediation' without defining the mediator's role or process, creating a critical failure point for resolving disputes. [dispute_mediation]
|
||||
- (0.90 · medium) The 'swap_listing' block only captures a 'required_skill' and 'estimated_time', failing to model the reciprocal nature of the skill exchange required for successful matching. [swap_listing]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.95) Requirement REQ-003 states that the system must ensure both participants rate the exchange positively, but no block or association is defined to model the rating from a participant (Knowledge Worker) to the exchange session. [req_003, exchange_session, knowledge_worker]
|
||||
- (0.85) The 'Knowledge Worker' block has a `reputation_score` property which is mentioned in the seed hypothesis as being portable from LinkedIn, but no association or constraint enforces this portability or linkage to an external source. [knowledge_worker, identity_verification]
|
||||
- (0.75) The 'swap_listing' block has a `required_skill` property which is not associated with the source of the required skill (the Knowledge Worker) or the type of skill being sought. [swap_listing, knowledge_worker]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (Knowledge Worker) and key concepts (Skill Profile, Swap Listing) necessary for a marketplace.
|
||||
- **model.accuracy:** 3/5 — Associations are generally correct but lack structural rigor; generalization/composition relationships were not explicitly modeled.
|
||||
- **model.parsimony:** 4/5 — The model is reasonably focused on the core exchange mechanism without adding excessive or irrelevant filler entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (no money, identity verification, dispute mediation) were accurately and completely captured in the model's constraints section.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., bounded listings, reputation usage) and directly challenge core hypotheses mentioned in the seed text.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive failure modes (matching logic, reputation portability, reciprocal nature) tied directly to the seed's core mechanics.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and question-led, effectively challenging the underlying mechanisms rather than just listing missing features.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run that excels in identifying critical risks and constraints. Minor improvements could involve refining structural relationships to achieve full accuracy.
|
||||
|
||||
---
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.20
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TP3DQW9148JlynJD-WAo_c105eqGFEMKU-YTwNK7vqUwMsK9ldj6zB0aUFsgggkxDwB8Uex1oD6d4p96QD4U3vpFoUroo0oJRa4TrIE7zEZorQS3T1Y4HeeDA08_1k3ztnJNOJUzAvaTyPEsAtnRrOLkcVFr2PUqS3DNu4TE-ecHeDfJk91wYvMvZVfILdfHDwMPdR-ekvdUEmAA9nqWKco9nPZ7yjLkiC6BmbTVBgm01Uo3VEkTm7ejWoZ5fe59RUUuRFyaPHmgThIQkMcCCHjAhZpx5m00)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclists (actor)
|
||||
- Associations: _none_
|
||||
- Constraints: _none_
|
||||
- Requirements: _none_
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The product will be accepted by the general group of 'Cyclists' without needing specific feature validation. [cyclist_thing, cyclist]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (1.00 · high) The model lacks any defined use case or specific problem for the target user, making it impossible to define measurable success criteria. [cyclist_thing]
|
||||
- (1.00 · high) The model does not specify what 'useful' means for the target user ('Cyclists'), leading to scope creep and undefined product boundaries. [cyclist_thing]
|
||||
- (1.00 · high) The model fails to define any interaction or relationship between the 'Cyclists' actor and the proposed system ('Cyclist Thing'), resulting in an ungrounded product concept. [cyclist_thing, cyclist]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (1.00) The systemOfInterestId 'cyclist_thing' does not match the ID of any defined block.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 3/5 — It correctly identified the core actor (Cyclists) but missed potential related entities like 'Bike' or 'Route', which are central to cycling.
|
||||
- **model.accuracy:** 4/5 — The structure is minimal and correct; no relationships were defined, so there are no structural errors to penalize.
|
||||
- **model.parsimony:** 5/5 — Given the vague seed, the model added only the bare minimum entities without padding or fabricating unnecessary blocks.
|
||||
- **constraint.capture:** 5/5 — The seed provided no explicit constraints, and the model correctly captured this by leaving the constraint list empty.
|
||||
- **asm.detection:** 4/5 — It successfully flagged the vagueness of 'useful' and the lack of defined use cases, which is appropriate for a failure-prone seed.
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, focusing on scope creep and undefined interaction, directly addressing the seed's ambiguity.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete in their critique of missing information, though they could benefit from more question-led prose.
|
||||
- **Average:** 4.29 · **Min:** 3
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLJRRjim37ttLn3fcr5dRdf1LmH1kwx3W073jZxiqQ1bEX4YIvueDmYA0lk8VU6-PFIj3TO4w8iWaeSy52-wve0-D9M9UArjZHuho56jbzurjhXrndauAi-ffDaEWbTOk8ssIoZHCErOb45cU8m0FhoAKfHOfgt5-OByeCqlyVtbJBGtthlDeBoWy-Aq4EKdPyDNh0WcdyayKD0A9z7J3fzrWGxGfNIgpWxGFKKhNH1GLOSjL5Jbv3cA-kGdTmzamttmXDK4a85QcJivuoq7gc0-xo-B1HpDr1bUvAFpD--AHf7lFEj143oguFnYCS9kdUMc6c1g45x3kbJW8mPyoIGhH1HeRfnQ2tJ4_jIU4kMg6eFE3S56XnLymNADBtd89JZb39HIZ4ellM2T1SQImhQZMBSEsKPYPMCiRRHKQJvlqmmUjGtox4UfuK1yBuCWVBcswBV7mBqew5BBW6X3n-0Roo14E5gja_8tDGG79scw7chyZabrMVIehA3Qk6tLrk0m_KsDQaKmcwPWj2BBr8N0JfqDgdt-qNDPacInhyjNK3UvqRmYpdfp5bpqtAs3NjpahBpEgSLdlPMpqhigseEFasItiLSWILgkdGyS7Up6T7g06-lQw8uuUbt4jtZzLxUtk4I6bcGBy7gv2h9nSVo3VZMIT4lASTohxhudQNeIn_3tznyOruIXQZY09qNwWRglf_R4t7YI8LXQ-LwwPieSUGv9tt9P_G2kX21j7oTwkaCo6qZ69KiABy4jbCPjshaCNZQWuysf2r9GYOsHc3Nr6u5YbmcTHlAi_d7HjHpoJ_u3)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Producer (actor), Consumer (actor), Event Data (block)
|
||||
- Associations: producer→eventstream_system:publishes_events_to, consumer→eventstream_system:subscribes_to_streams_from, eventstream_system→event_data:transports
|
||||
- Constraints: Wire-compatible with Kafka producer protocol for migration safety, Single binary, single config file, runs on a single node up to 100k events/sec, Apache 2.0 license
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The target user group (Platform engineers) will prioritize operational simplicity and ease of adoption over maintaining perfect feature parity with existing systems like Kafka. [single_deployment, kafka_wire_compatibility]
|
||||
- (0.85) The system's success hinges on the assumption that platform engineers view backpressure as a critical, first-class protocol feature rather than merely an operational concern to be solved by configuration. [req_001, req_002]
|
||||
- (0.75) The product assumes that platform engineers are willing to adopt a new streaming system despite the operational risk associated with abandoning established industry standards like Kafka. [kafka_wire_compatibility, single_deployment]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The requirement for producers to receive backpressure signals as a first-class protocol feature (req_001) conflicts with the constraint of being wire-compatible with Kafka producer protocols, which do not natively support in-band flow control signaling. [req_001, kafka_wire_compatibility]
|
||||
- (0.85 · medium) Maintaining single-node operation up to 100k events/sec (single_deployment) while supporting the complex, stateful backpressure logic required by consumer-driven flow control (req_002) may introduce significant performance bottlenecks. [single_deployment, req_002]
|
||||
- (0.75 · medium) The model does not specify how the system will handle message ordering guarantees when transitioning from a distributed, partitioned Kafka environment to a single-node, flow-controlled architecture. [eventstream_system]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The requirement that producers receive backpressure signals (req_001) implies a direct communication path or state observation between the producer and the system, which is not captured by any association. [req_001, producer, eventstream_system]
|
||||
- (0.90) The requirement for consumer-driven flow control (req_002) implies a dependency or signaling mechanism from the consumer to the system, which is not captured by any association. [req_002, consumer, eventstream_system]
|
||||
- (0.80) The constraint 'Single binary... runs on a single node up to 100k events/sec' (single_deployment) is an operational performance claim that cannot be validated or enforced by the current structural blocks and associations. [single_deployment, eventstream_system]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model correctly identified the core actors (Producer/Consumer) and the system of interest, though it missed modeling the 'backpressure signal' itself as a key concept.
|
||||
- **model.accuracy:** 3/5 — The associations are structurally correct but lack semantic detail; for instance, 'transports' is too generic for event data flow.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and only includes necessary components (Producer, Consumer, EventStream) without adding filler blocks.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were perfectly captured in the model's constraint list with appropriate application scope.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., prioritizing simplicity over parity) and tied to core business drivers mentioned in the seed text.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific conflicts between requirements and constraints (e.g., backpressure vs Kafka wire compatibility).
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, focusing on technical contradictions rather than general business platitudes.
|
||||
- **Average:** 4.29 · **Min:** 3
|
||||
|
||||
> This is a very strong run that excels in identifying deep, domain-specific risks. Minor improvements could involve modeling the backpressure signal as an explicit data type or concept.
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/hLNRRjim37ttLn3jGu0W2RBqWYu8YhQxe7jPks7Ogq5Bj2DKbZn9TfSL0VOH-y9zoIZdvhOf-hAN8AH8diD34hxm0LseIvtuEsKgT5X2XkAkSBOsyftLri5XVa8vJJeHVexItYjJG8xQK-T5QFGU7XA0gsTLP8xvEBvO9ycjlTaPlZsRiFVIENk_TfxIYJoIxFneJVYC9S71DUc6WX9ua2mxUCO6UW5kJ4VY-0MuPJ9Nae3AAYoWf38ZvvDaHVxWbW9ySsYync3T0Q27Ya8QjYwOpVp21ohFp-5m8exnD6ivA9EJ8oCeBQqadRAseVOm1J9r-Q2afa7bh23lMRg18vIB9QTfAp06MuS5fG8h58ffJO6pc1YJxH2xkJUqusIZnNHGS3QpYInQhekADhZI5WE18IBHpugSQXbgQaXpY0-EoUmmBckfB7moLHrQ58nsggBDE1bplLlZD96s-eL1MPDA3BYh1k18iEpyFvMrQqFszFETPz35QgAz5okpSf5gD4KT9NeAblCcKCemz89eLpJ7Hl7WtdFcLhnixLtLUmn8fOsaSJN4hPQLnaLUwu6edRAr7s1Hk3ZEXbXYNIxR9GZaAaU1P5ho6nFkohoXTME6aXebQBSra3F3oga6nQ8b6NLjOz9Tp6mchE5go3WjrssZrzQG3u2PPVd2d42gV9ELKRWdCj1RzxMwKvNpFKMp1sWaz3OR7ZqjWNbRCfNA2srzxVR29xj570xFdvxA56fFFkckqTwed4Xw3WWswQx1Qx5FtZRXZqSxXOAO7i_H9_iq7OtsW61LQKK-PP3dChoMaMXYI0bE5VF0Iz7lVwKVjN9KaWc-tr-vFdmPZiTl-dtu-_iFt3eR6-UBusbndGOpJUrSM43UkTrYhSx9vksGcKNFFxh5KQVuv7yKZyS7c-CR9TpNghlaWd_uq_4F)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Audio Input (block), Video Input (block), Pet Behavior Data (block)
|
||||
- Associations: pet_owner→pet_translator:uses, audio_input→pet_translator:feeds_into, video_input→pet_translator:feeds_into, pet_translator→pet_behavior_data:generates
|
||||
- Constraints: On-device inference for privacy, Honest about the difference between 'emotion classification' and 'translation'
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that users will perceive the output of emotion classification as a 'translation' despite explicit warnings. [pet_translator, honest_disclosure, req_001, req_002]
|
||||
- (0.85) The system assumes that the combination of audio and video inputs will yield sufficient data to generate a 'believable' interpretation. [audio_input, video_input, pet_behavior_data, req_001, req_002]
|
||||
- (0.80) The system assumes that pet owners will find the generated interpretation 'useful' enough to justify using the app repeatedly. [pet_owner, req_001, req_002]
|
||||
- (0.75) The system assumes that the pet owner will be willing to provide continuous audio and video inputs for analysis. [audio_input, video_input, pet_owner]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on combining two distinct, complex ML modalities (audio and video) for inference, which significantly increases the technical risk of achieving reliable joint interpretation. [a2, a3, pet_translator]
|
||||
- (0.95 · high) The requirement for on-device inference ('on_device_inference') restricts the use of highly accurate, large-scale models necessary to achieve a 'believable and useful interpretation' (req_001, req_002). [on_device_inference, pet_translator, req_001, req_002]
|
||||
- (0.85 · high) The model does not define the technical mechanism or data source for generating 'believable and useful interpretation' (req_001, req_002), making feasibility unquantifiable. [pet_behavior_data, req_001, req_002]
|
||||
- (0.90 · medium) The 'interpreted_message' (part of pet_behavior_data) risks being perceived as a literal translation, undermining the stated constraint ('honest_disclosure') and leading to user mistrust. [pet_behavior_data, honest_disclosure]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-001 claims that the interpretation of vocalization is satisfied by `pet_behavior_data`, but this block does not receive direct input from `audio_input`. [req_001, pet_behavior_data, audio_input]
|
||||
- (0.90) Requirement REQ-002 claims that the interpretation of body language is satisfied by `pet_behavior_data`, but this block does not receive direct input from `video_input`. [req_002, pet_behavior_data, video_input]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model successfully captured the core actors (Pet Owner) and inputs (Audio/Video), missing only a direct representation of the pet itself.
|
||||
- **model.accuracy:** 5/5 — The structural relationships are clean; associations correctly link inputs to the system and outputs from the system.
|
||||
- **model.parsimony:** 4/5 — It is slightly padded with generic properties (e.g., 'local_storage_capacity'), but overall it remains focused on core components.
|
||||
- **constraint.capture:** 5/5 — Both explicit constraints were perfectly captured in the model's constraint list and applied to the system block.
|
||||
- **asm.detection:** 4/5 — It correctly flags the wishful tech assumption (perceiving classification as translation) and the data sufficiency problem.
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, focusing on technical feasibility, modality combination, and conflicting constraints.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, particularly in the risk descriptions, but occasionally use slightly verbose phrasing.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a strong run that effectively tackles the 'wishful tech' nature of the seed. The detection of technical risks (modality combination) and conflicting constraints was excellent.
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLRRZjis47tNLmmsBubHBrnl4gIBO94cQ9ENf4tIFXCKEP88Kno5fEmwmGBzY7vXlwH3IefLNv8Nhp73uJayPsQyBqAKFlQjBSB6k4vwsK8frQRsr3lzYYnvU50zmGhNinEXaPfsnjLGIHjmbb5MXW1V2e2N9xVeIbQhb26lqO_HyfdywTcQepzxJxin-1IVw1lDmL_9nNUoHRXwWtQBqIXvLTpFy1n5l02tmXlr-0BSVT4OZO1j5_VGObkY3qKnaBzwtnkCy8Pw7-3ZFaHihq06-9J2eidXvyz3JjpTmOErUYoVbfb4vvcEYgX5D2sAO2a6k8KGFMl5UOSrHIEZ8ISQ4oBv_I5zF-7_6T1dn9w_C9HKV92H4eAI5fsMNaYbCAJBqVNjbvgejYYcx0_KHqkq4RMNNNFFTHgn4plo6z5vDjEZKtW6-jLu0Nn4lpKACuslc648CgNIajgCb6Hd1ESxCYxE7oktqbXP6clYNcWP9ITB8elI7G3V8flAVVG1F_KOOiPhXvZmG-mOBaP6GnIM1PpZiVWTXL7xCs_xVJ87NgRhCjJ1i0nn19Ldu-WwOC5F8IQC3oZrSkSDD-NhR0byLLGw3RyDji33jqPv2bH5U9qSUWIi7R9PgJCoiID3nIpF91KvXfOSp4mFQlOiYSVGyPiWSVnC3a6RQdGy0FxLMQDCj7k672NZ4SwOKu6ORhWCz8v0zf5Q9gGWIvPCt9d8un67lkNIolXse5pnxCK6WIFCy6O5Mku3F2nxOpL4pxGVPGh66PuFAt98TEWDwNCqYWonNDxzRofl1z2wvmSKyqGwUDB5jwoKr46mCI9R5ehZpZi5C847YDBefPxPdwt8tFBHW8xxXqwu7BTTccAMYAyjlcNyzVKn7TcndHWInlSAHrQpYXE_qx7Z1ts_uYAZisP-iwAmM3DxyAPk8e_QOf7sWl7Oif1XiHX2lxnVhbO_BXRmxz__m1_SLSCkXZPrFxh0zkTcoo_sQ8qikTrhv5qi8_3-uMFhr2gHMuNwa9VZCYt7ePLvBN3X3d5pFGDSNm2STZeUN9skCMvhWadWvNwSjJGcqaRqZftPuYHtdZsUm8O9PY9fO4gA3V3FN6HrvankBZ3XpRvbdGvlhporJ4BrFgqfw8XNCV-2Mfw5piJqbxXCzomNGL0DwjwcWscAU4j4HcKt1a-A5_p1_oVy1m00)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours System (system), User (actor), Calendar Service (block), Meeting Request (block), Protected Block (block)
|
||||
- Associations: user→quiet_hours:configures, quiet_hours→calendar_service:reads_and_writes, meeting_request→quiet_hours:is_processed_by, meeting_request→user:is_sent_to, quiet_hours→protected_block:enforces
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The tool assumes that users will accept a system that negotiates protected blocks invisibly, without requiring them to manually approve the changes. [req_002, initial_trust_period]
|
||||
- (0.90) The system assumes that negotiating alternative meeting times will always result in a mutually acceptable outcome for all attendees. [req_002, meeting_request]
|
||||
- (0.85) The product assumes that the user's deep-work preferences are stable and can be reliably configured via a single 'user' profile. [user, a1]
|
||||
- (0.80) The system assumes that the user will continue to use the tool even if it requires complex integration with multiple calendar services (Google and Outlook). [google_calendar_integration, a2]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.80 · high) If the system relies solely on proposing alternative times, it may fail to address existing meeting commitments that are non-negotiable or mandatory for the user. [meeting_request, quiet_hours]
|
||||
- (0.90 · high) The system's ability to negotiate protected blocks invisibly (req_002) is constrained by the need for explicit user confirmation during the initial 30 days (initial_trust_period), creating a potential conflict in core functionality. [quiet_hours, meeting_request, initial_trust_period]
|
||||
- (0.90 · medium) The system must handle the complexity of coordinating deep-work blocks across multiple time zones and conflicting calendars (calendar_service) without generating ambiguous or incorrect scheduling proposals. [calendar_service, timezone_respect]
|
||||
- (0.85 · high) The model does not specify how the system will handle or prioritize conflicts between multiple protected blocks (protected_block) when they overlap, potentially leading to calendar corruption. [protected_block, quiet_hours]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) Constraint 'google_calendar_integration' applies to both the system and the user, but calendar access is a property of the user block, suggesting that the integration capability should be modeled as an association or a property on the user/system. [google_calendar_integration, quiet_hours, user]
|
||||
- (0.90) Requirement 'req_002' states the system must negotiate protected blocks invisibly by proposing alternative meeting times, but there is no explicit block or association defining the mechanism for generating and tracking these invisible proposals. [req_002, quiet_hours, meeting_request]
|
||||
- (0.95) The association 'a5' (quiet_hours enforces protected_block) suggests the system actively imposes time slots, which contradicts constraint 'initial_trust_period' that forbids automatic blocking without user confirmation. [a5, quiet_hours, protected_block, initial_trust_period]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured core entities (User, Calendar Service, Meeting Request) but missed key actors like 'Colleague' or 'Meeting Organizer'.
|
||||
- **model.accuracy:** 4/5 — Associations are generally correct and logical; the structure is sound, though a generalization/composition relationship could clarify dependencies.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused on the core problem domain without adding unnecessary or fabricated blocks or entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were correctly captured and applied to relevant parts of the system model.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., stability of deep-work preferences) and tied directly to functional requirements or constraints.
|
||||
- **risk.detection:** 5/5 — Risks are highly substantive, domain-specific failure modes (e.g., non-negotiable meetings, conflict between trust period/enforcement).
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, identifying specific conflicts in the model rather than just listing general flaws.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run. The detection of high-severity inconsistencies (especially around 'initial_trust_period' vs. 'enforces') demonstrates deep domain understanding.
|
||||
|
||||
---
|
||||
498
phase-0/reports/2026-04-28-FINAL-run.md
Normal file
498
phase-0/reports/2026-04-28-FINAL-run.md
Normal file
@@ -0,0 +1,498 @@
|
||||
# Phase 0 corpus run · 2026-04-28T20:01:12.358Z
|
||||
|
||||
**Model:** `google/gemma-4-e4b`
|
||||
**Ran:** 10 · **OK:** 10 · **Failed:** 0
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 4/3/3/2 | 1/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 02-habit-coach | medium | 5/4/3/3 | 4/4/3 | 0.90 | 4 | 3 | 5 | 5 | 4 | 5 | 4 | **4.29** | **3** |
|
||||
| 03-redline-ai | medium | 5/5/4/2 | 1/4/3 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 04-skillswap | medium | 5/4/3/3 | 5/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 05-cyclist-thing | failure-prone | 2/1/0/0 | 1/3/1 | 0.20 | 3 | 4 | 5 | 5 | 4 | 5 | 4 | **4.29** | **3** |
|
||||
| 06-eventstream | hard | 4/3/3/2 | 3/3/3 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 07-pet-translator | failure-prone | 5/5/2/2 | 4/4/3 | 0.85 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
| 08-quiet-hours | medium | 6/6/3/2 | 4/5/4 | 0.90 | 4 | 5 | 4 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 09-carbon-coach | medium | 5/4/3/3 | 3/4/3 | 0.90 | 4 | 4 | 5 | 5 | 4 | 5 | 4 | **4.43** | **4** |
|
||||
| 10-telemetry-lite | hard | 5/2/3/3 | 4/4/3 | 0.90 | 4 | 3 | 4 | 5 | 4 | 5 | 4 | **4.14** | **3** |
|
||||
|
||||
**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed
|
||||
|
||||
**Passing:** 10 / 10
|
||||
|
||||
## Per-seed detail
|
||||
|
||||
### 01-aristotle — Aristotle
|
||||
|
||||
- **Difficulty:** easy
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRjim37ttLmZfMz04QNf1LmH5kwB5zj2XwxfdWxRfH8iiUQAK82W2x2FsXVkIKRAJE6ZJbw0XgNCEoHFsXXnOvokLq5pg6YnK8eDyFhN6w-BEA6F5KNcE9Oux5JI3mYobdeeI564daoiW4g-949_Ve1GbbAEGCRP0sqQpI_XqEURehRLcsGOlyBmuBJZuOBJx1XMAtXTK2tGoXrwoxl1fu_00tGXFyxC3TEja9WiKMDLk9IgiChIK98tutgsLv8nJs1D00ZRVn6H2At9ONL-Bet5-1XTPP9QQd7JUIQD1fLcG07ObhaHcZ4BGNABCCkL1pv7IoZ3lBhVUaFxKB79geV307Vrmla3jeW3VpJ1CpB0Oo9snh2Leo8stX4jZvu77UM8ArBvwHJs5AHR7XQJsp_MEysxxIZmojfMW8cC7hMhZp9SfayzRFlBPBynT2XQ1gSXPDWF7QsioXLLAw7RXBT_BJEfvSCqJUqwZ5LyretCVFBf8xDg2j6uBzbaPdHUMkbMDc_voetEidGTr33p-LN28EhPOAfOd5xYE2zgofBEmpBYThRYDkaTFJcXSCAtnhlPE04-YgXNoGiYek67X38V3Dh3OT1kbL_mwZQ_JvdMwVJAPi4gc0nOIEzaEv1cfvXGAnMFM-QhPjJWPZaaydOy2MMsHq2vGb6oYoDJMfZKN78P-k7z-kXLS2G7ttv-_dQssNR815qZx_XKR_qQg4cqDwGRa7RBaVNCE1jUTtysLy8IKx08X_Px7hi94SwHGVHZuZKc25OArALrADqi-sCHmkAyBwbf9VkTCqftg1sNx4_cWC54uHLq8AwSpnoRizv_njvSMA-w3-lqcTFzzC1gTzFjnGIypzbcyCD5DV5jpgQGBzy3YeEanl1QXpUuwDmCSTa377T3E3w11pZ2Sf45paWImJEwTyPXWEzx8mWCCErd88Xvx6Hm2cfPyBeS9jzSqcTpm1_yF-Gy0)
|
||||
- **Post-validate:** stripped 4 refs, dropped 2 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Aristotle (system), Undergraduate Student (actor), Coursework Material (block), Thinking Partner Interaction (block)
|
||||
- Associations: undergraduate_student→aristotle:uses, aristotle→coursework_material:processes, undergraduate_student→thinking_partner:engages_with
|
||||
- Constraints: Must never output a complete solution to a graded problem, Response latency under 1.2s P50 to preserve flow, FERPA tenancy — coursework never leaves institutional boundary
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.85) The system's refusal to provide solutions will be perceived as valuable by students who are currently disengaged. [req_002]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The FERPA tenancy constraint (ferpa_tenancy) creates a high risk of data leakage or non-compliance if the underlying infrastructure cannot guarantee that coursework material never leaves the institutional boundary. [aristotle, ferpa_tenancy]
|
||||
- (0.80 · medium) The latency constraint (latency_p50) may be difficult to maintain if the system must process complex, multi-turn 'thinking partner' interactions involving detailed history tracking (interaction_history). [aristotle, latency_p50, thinking_partner]
|
||||
- (0.95 · high) The 'Must never output a complete solution' constraint (must_never_output_solution) requires complex guardrails that could fail, leading to the system inadvertently providing an answer and undermining the core value proposition. [aristotle, must_never_output_solution]
|
||||
- (0.75 · medium) The model lacks a defined mechanism for how the 'thinking partner' (thinking_partner) will guide the student through the material, risking that interactions become generic or fail to address specific problem sets (problem_set). [thinking_partner, coursework_material]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-002 states the interaction must be low-stakes and encourage thinking without providing final answers, but the 'thinking_partner' block is not associated with any requirement satisfaction. [req_002, thinking_partner]
|
||||
- (0.75) The 'undergraduate_student' actor is associated with the 'thinking_partner' block via association a3, but no requirement explicitly mandates this interaction. [a3, req_002]
|
||||
- (0.80) The 'thinking_partner' block is defined as the mechanism for interaction but has no associations linking it to either the student or the coursework material. [thinking_partner, a3]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured core actors (student) and concepts (coursework, thinking partner), but missed explicit mention of the university/institution as a key boundary.
|
||||
- **model.accuracy:** 4/5 — Associations are generally correct (uses, processes, engages_with); however, the relationship between 'thinking_partner' and 'coursework_material' is missing or weak.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused on the core problem elements without adding unnecessary filler entities or blocks.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (FERPA, latency, no solution) were correctly captured and applied to the system boundary.
|
||||
- **asm.detection:** 4/5 — The assumption is specific and tied directly to a core requirement (low-stakes value proposition), making it actionable.
|
||||
- **risk.detection:** 5/5 — Risks are substantive failure modes (guardrails failing, data leakage) that address the constraints and domain specifics well.
|
||||
- **voice.character:** 4/5 — The findings are highly concrete and technical, maintaining a skeptical tone while remaining focused on system boundaries.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
---
|
||||
|
||||
### 02-habit-coach — Habit Coach
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/dLNRRXCn47tVhnYL1o1ggj820bHLGBbKuXOeF5kptjbTArxFOZiD5KJY8_X2leInDuLjINZW9Kfcx7Dcpfnn7eU8FYuwMuIvSJrwxA14FMyyBrnrmfOztAZlKKq7enEXnOgNnZLGemqqocYB8S3N0k3fNoXLZVL-oh2loA-ZvGDy-E10ea-yv-KwU9_kLOUL15-mYs-n8zWv9NjEqMZSALOZFiUHjj3jqw6-kuLkLRIc8g2kZnVGKLUI3qKn5BzpYgM9SCAesnt006twhNJw3KT7uI94wewFuSQ1leltoynUOKI5JUEfmMZOANADSGIFe6IsX4xEIC7aegemj2MZhqR9rINrB9_9hCDn8KGTsGjVucaOhGeKesYlDFPIcJNImWQeazROAYFfzEqFqEM1C-s977mq8kypgJ-V3JceeWHLQagkbPRrVCsFVQyM0HjIbXjX3T5B9PB85qAV6kp9ApadMRT8mevAOdSkGcU5OEPPr1yA7CLLd-FNU2FtHYjH10MM1FThi4JzRj2jzBraFzzjkImDXTNbDBnuLXJKzAVCrUyHlrc42EaQe6NNX6G6y-uCnfgWg-0bSsC9Nc1dx0MyDcw-com7ddgB0XrRwd8BdG0guo8r7YCf-c92db6Kye8q8orA4IWdSaTNfvGBpX1GqdhmKxaHFLjB5OYYSF9kDdi2jpwTtHuAUpRxC7q5jvv_kftvSykzD-UeByIgNMyDfjwtCxzb2AmDMgW9uy9JW5esP6wg03-__v0sGGmbcvFD6bBVq2zAQtHQ_wHd9dQi1XZr6sO3Px5sCq-dutKg1cjUZuuNBkMkc_8IxUYR93UOf_ZtDFRsZg_izADaQolUaNonJSZ_llJNHbsvkK6-Vp8LbXgI-NlJj56iFfbyeCyBuwaJGS9aCeIUlv_kxz-PJF8WFxO4mwCrMAn6dHuD4GOGvCMCwQ5Q4it5uMCnnOjncRPXhNQoNPxNteZaO0l9y1Id_TW8RLms4GtM7mCURW4ChQbZnXlAkHcaTbdDMYOkBu9OLykMH1OUQx5a6JpxlKAq6XGh7ik7_C_z0W00)
|
||||
- **Post-validate:** stripped 1 refs, dropped 0 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Habit Coach (system), Parent (actor), Child (actor), Screen Time Data (block), Conversation Prompt (block)
|
||||
- Associations: parent→habit_coach:, child→habit_coach:, habit_coach→screen_time_data:, parent→conversation_prompt:
|
||||
- Constraints: Must work alongside iOS Screen Time and Google Family Link, not replace them, Child data must be parent-controlled per COPPA (US) and GDPR-K (EU), No social features for kids — no leaderboards, no public sharing
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) Parents will adopt a tool that shifts their role from enforcer to collaborator, provided the data is presented in a non-confrontational way. [req_003]
|
||||
- (0.85) The system assumes that parents and children will view the data presented in the 'conversation_prompt' block as a neutral basis for discussion, rather than simply another source of conflict. [req_002, conversation_prompt]
|
||||
- (0.80) The system assumes that the existing OS-level tools (iOS Screen Time and Google Family Link) will provide sufficient, structured data ('app_usage_log', 'time_spent_per_app') for the 'data_aggregation_engine' to function effectively. [must_integrate_existing_tools, screen_time_data]
|
||||
- (0.75) The system assumes that the 'parent' actor will be capable of defining and maintaining a consistent goal-setting capability without requiring constant manual input or oversight. [parent, goal_setting_capability]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) Relying on external APIs for 'Screen Time Data' (a3) creates a single point of failure if either iOS Screen Time or Google Family Link changes their data export schema, breaking the core functionality. [a3, screen_time_data]
|
||||
- (0.80 · medium) The 'Parent' actor (parent) is given a 'goal_setting_capability', but the model does not define how this capability translates into actionable data or prompts for the 'Conversation Prompt' (a4), risking feature bloat without core utility. [parent, conversation_prompt, a4]
|
||||
- (0.90 · high) The 'Parent' and 'Child' actors are both aggregated into the 'Habit Coach' system (a1, a2), but the model lacks mechanisms to define distinct user roles or permissions within the parent_dashboard, risking data access conflicts. [parent, child, a1, a2, habit_coach]
|
||||
- (0.95 · high) The 'data_aggregation_engine' within the 'Habit Coach' block must reconcile disparate data sources (iOS/Google) into a unified view, which is technically complex and could lead to inconsistent or incomplete data presented in the parent_dashboard. [habit_coach, data_aggregation_engine, screen_time_data]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Association a4 suggests that 'Parent' generalizes to 'Conversation Prompt', which is semantically incorrect as the parent does not generalize into the prompt itself.
|
||||
- (0.85) The requirement REQ-002 states that shared data must ground the conversation, but no block or association explicitly links 'conversation_prompt' to 'screen_time_data', creating a missing satisfier.
|
||||
- (0.75) The system requires the parent to set goals (parent block property) and facilitate conversations (req_001), but there is no explicit association or mechanism showing how 'goal_setting_capability' influences the conversation process.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured all major actors (Parent, Child) and core system components (Data, Conversation Prompt). It missed explicit mention of the 'data aggregation' process itself.
|
||||
- **model.accuracy:** 3/5 — The generalization association (a4: Parent -> Conversation Prompt) is structurally incorrect. Other associations are plausible but lack clear labels or justification.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and avoids padding with unnecessary entities; every block seems relevant to the core problem space.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (COPPA, integration, no social features) were correctly captured as blocks/constraints applied to the system.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific and tied to technical dependencies (OS APIs), which is strong. They avoid generic platitudes well.
|
||||
- **risk.detection:** 5/5 — The risks identified are highly substantive, domain-specific failure modes (API changes, data reconciliation) with high severity.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, focusing on technical dependencies and structural flaws. The tone is professional but could be slightly more questioning.
|
||||
- **Average:** 4.29 · **Min:** 3
|
||||
|
||||
> This was a very strong run; the risk detection was excellent and highly specific to the domain's technical challenges.
|
||||
|
||||
---
|
||||
|
||||
### 03-redline-ai — Redline AI
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRRjj647tdLmoSbrQ835Uv81K28wxQj2wS14qCz74nt1rA2--5tLrQ4G83_OX-ORyaiqlQfdLx4SIPENFcSePy5nE6r5bJnHljMWneeKPvimo-SshXZG_mh7b53Sr64N65og-rMqA39jB88mt62DygW5zsigW6cxFiyK5H6Apr6_pvpOoj5o7uzM1yJQ_K2yN6zzwbZsW9JluWSqj9IpofxaPypYSwG7T6B-JB0tHtrKeh0h9jse0bMrE8LTKNV_APbD6EuEBo1312w1y5QdZxDcvY8dj-3izcyYM-hWktTecM0PFsJhG6K-E3XJd457X4xEUyNIHXT1qmR0HAISmpXzfxG-Yuce7vYfPeu9hG5cQJ7qNYHsP6cNnWuamO6CTuSftzzZiv2enQSE-HpFC_EwUvquiOlTIOw7cVz5DBVOKHhWZLtIFfWcSSE3lywcLdoQN2BGUhK8ELIwYDbpT32Jo5t2g9j6dfIRCCv70k-sjwj8_OR8jk0uio4xZgPp9G5fTuEgujtjuPnuoM4YfCk8VjVfLVkkMIOcx_oKhZWtcBPN2G4dA5RlcamK0OlUF_eiMqscgzOLavoPcb3nd56s9jSK7t5NteOdeG3C5QSueFMWOVVPFWRzPggT1oc7W84sksSubCm6J8zbBdxWg7uHSuEBgb02KUCa6SIZPxoqRh5Pa8FxH1tsgM6MMafE3YZuNUUL4GuXuXIffyyOUhkEYKJeNFW6yWiszWcDzeqgMsce_lQ-j3mjfGeSICBZH6FBgEq7tvj80PN6VvNLuMyHYDt0GqmLjGk07l-jHy2JCn2jYJinhTsdHwFZxwEMJ1LIDB3jYzZpcqmUVBtXUyLzbpfcaePC7iGFP8T0xBSkk9etVf3u9q51R33MkvtbJxT7LwkjKljZmfYYBvweWGZi4EBFCOP7j9nsBpjiWf27gvIdnuaybd-gVJWV8aucJIctxxQtfszjDa0l__-n-yro6cQPkLCsIDh0--NNGAUGZt0-rFhvSJyO7oxLZjEfxvwIZhRCXwlIBOUW7t-42tcjRy2HafFt8oVgUWojU1NLhv_DvuBAVeDafUZ5oTLjnVtsNrZd_u2_qT)
|
||||
- **Post-validate:** stripped 4 refs, dropped 3 findings
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Redline AI (system), Legal Team (actor), Contract Document (block), Company Clause Library (block), Redline Suggestion (block)
|
||||
- Associations: legal_team→redline_ai:uses, redline_ai→contract_document:processes, redline_ai→company_clause_library:references, redline_ai→redline_suggestion:generates, contract_document→redline_suggestion:is_marked_by
|
||||
- Constraints: Must integrate with Microsoft Word, Contract content must never train cross-customer models (privilege concerns), Audit trail of every change suggestion must be exportable, SOC 2 Type II compliance from day one
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The model assumes that integrating with Microsoft Word (must_integrate_word) is technically feasible while maintaining strict data isolation and compliance standards (no_cross_customer_training). [must_integrate_word, no_cross_customer_training]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Achieving the latency target of 'minutes' for first-pass redlining (req_001) while maintaining data isolation (no_cross_customer_training) and integrating into Word (must_integrate_word) presents a significant technical challenge. [redline_ai, req_001, must_integrate_word, no_cross_customer_training]
|
||||
- (0.85 · high) The model does not specify how the 'reasoning_path' in a redline suggestion (redline_suggestion) will be generated or validated, which is critical for building legal trust and meeting audit requirements (audit_trail_exportable). [redline_suggestion, reasoning_path, a4, audit_trail_exportable]
|
||||
- (0.95 · medium) The system's reliance on the 'company_clause_library' (a3) implies that if this library is incomplete or outdated, the redline suggestions will be inaccurate, undermining the core value proposition. [redline_ai, a3, company_clause_library]
|
||||
- (0.70 · medium) The model lacks a defined mechanism for handling conflicting redlines or suggestions when multiple clauses in the 'contract_document' (a5) are implicated, risking legal ambiguity. [redline_suggestion, a5, contract_document]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Requirement REQ-002 states the General Counsel must review AI suggestions, but no association or block property explicitly models this required human interaction flow. [req_002, legal_team]
|
||||
- (0.85) The 'redline_ai' system block lists 'integration_platform' as a property, but the constraint 'must_integrate_word' only specifies the platform (Microsoft Word) without defining how this integration is structurally represented. [redline_ai, must_integrate_word]
|
||||
- (0.75) The requirement REQ-001 (completion in minutes) is satisfied by the 'redline_ai' system block, but no specific mechanism or property within the model defines how this speed constraint is met. [req_001, redline_ai]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (Legal Team) and system components (Clause Library, Redline Suggestion) needed for the domain.
|
||||
- **model.accuracy:** 5/5 — Associations are structurally sound; 'uses', 'processes', and 'references' correctly map the relationships between blocks.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean but includes some generic properties (e.g., reasoning_path) that could be more tightly defined.
|
||||
- **constraint.capture:** 5/5 — All four explicit constraints were correctly captured and applied to the system block in the SysML diagram.
|
||||
- **asm.detection:** 4/5 — The assumption links technical feasibility (Word integration) with compliance needs, which is specific and highly relevant to the domain.
|
||||
- **risk.detection:** 5/5 — Identifies critical failure modes related to trust, data lineage, and core inputs (clause library incompleteness).
|
||||
- **voice.character:** 4/5 — The findings are highly specific, technical, and skeptical, maintaining a strong PM/Socrates voice.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very high-quality run that successfully navigates the complexity of domain constraints and failure modes.
|
||||
|
||||
---
|
||||
|
||||
### 04-skillswap — SkillSwap
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZPNRRjD048Rl_HIZSWDHKvKMKAbGnLaWSIuIbwknFOwNxC7iZXC2giH3y8Gy2RDhjqre8d7ZmwndlZdyktuS6GFtrXHnhbs70IsKMCrdmVUkVkQD3t2hkKyD7Qrz4Lki_LAx6JHe8gsjL0PZX9y5mDCRKUe6cyEquaDDORIM9_Zmv4YiJqBmoz7uWExNnxKONth7xz0IxBqYio3M5UuL5siyvvbsu0xfkBgt0tTHjBec8Dln2YpPaa8iYY7vlVEvDkPyYHryCCYD3tOFC49CvYZcZeAAgyXauT6ZuU7i36uTLVVmGPcJeUzLYsv6gjMHVLZ1AKGEqYzPuz17Lh7oWSJi-iIMV4Rs9xAfeYy-p2baRCWMjSmM0M8b4OMNE1R5o5WQKaDsww1iKRX0RR3KHdDAe_JU4Bfhu3c5XQu8tZSD9SUXrC6e_6WKR6byDH-nGsHUTRJEeyZQ8bEjvBwjjjpIjyXLEq3aLTdq-a_uGDzw7IHEvcoKb5mMQFhjhLjeMi9v7iZOkcIvdjK613aLTXK6Bhzc-ixNp8ihLRppOBsZ5GoZZI2tEe9sCIc9Mu9kNItEg_ovkjI2y2ll92ngnpc9gynVr-GupMSXtMv4uwozWuL6UAFTdEhN3aIxKLkPP01aUPovliiSFVgg3T_ThESwTZrB-bKISWqiPL6jKKGrTW3-_FejwQSwmupIOG1f8D9s8MLcFSHGW-CENh4ftEdqREiEEXNgJ7Q9R43L1TzeqV9ISxlD_LzLda8LI1GNbP48Of6bOXiaUr_BxHHcv2XapquWqa0qmvgI0Mv1Rof9rD6NLdDKerr6LTmS-C71hfBbrMY1io-spo-vRcQnuNHp0F_5AWn9uJK4FMjP53-PV1gscfKSucGoc5vyd1uUtfrCiWu-Ywx7K5REBJdlgtIKIEj08atPJrF0ojlEK4x4Du139hdZS3ktLMQp3tBcTVdXiaFxuowySx16Fjf19fVEEU2bJtipkOUOr3koAM4JC8b7InwEO4aqJmbzzTe9Pfrol8FI1Us3Pls3e9KkJVDf3PSroAynofwHCZiVyhahKv9933EeaV6Wa7uCRIuUoqN-h7y1)
|
||||
- **Notes:** SkillSwapPlatform is the SoI; RemoteWorker is the primary actor; ServiceOffering and SwapMatch are core blocks.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: SkillSwap Platform (system), Remote Worker (actor), Service Offering (block), Swap Match (block), Review System (block)
|
||||
- Associations: remote_worker→skillswapper_system:registers_profile_with, remote_worker→service_offering:creates_listing_of, skillswapper_system→swap_match:generates_potential_match_for, remote_worker→review_system:submits_review_via
|
||||
- Constraints: No money changes hands inside the platform, Identity verification via LinkedIn or similar at signup, Disputes capped to mediator review — no chargeback equivalent
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The platform assumes that remote workers will actively use and maintain their 'marketable_skills' and 'skill_availability' properties to create listings, which is necessary for the core function of generating a match. [remote_worker, a2]
|
||||
- (0.90) The system assumes that users will accept and rely on the 'trust_score' property within the 'skillswapper_system' to guide their matching decisions, even if they are unfamiliar with how the score is calculated. [skillswapper_system, req_003]
|
||||
- (0.85) The system assumes that users will engage in enough 'Service Offering' listings to ensure a sufficient supply of skills for the 'swap_match' block to function repeatedly. [service_offering, a2]
|
||||
- (0.80) The platform assumes that users will be willing to complete the 'Review System' process after a swap, which is critical for building the necessary reputation data and trust. [review_system, a4]
|
||||
- (0.90) The platform assumes that 'Identity verification via LinkedIn or similar at signup' will be sufficient to establish the high-trust reputation required for matching. [identity_verification, remote_worker]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on 'trust_score' and 'review_system' to manage reputation, but the model does not specify how portable external reputation (LinkedIn) data is integrated or weighted against internal platform performance. [skillswapper_system, review_system, req_003]
|
||||
- (0.80 · medium) The constraint 'No money changes hands inside the platform' combined with the lack of a defined payment or value exchange mechanism makes it unclear how disputes are resolved or if users can be compensated for time lost during a failed swap. [no_money_exchange, dispute_mediation, skillswapper_system]
|
||||
- (0.95 · high) The 'swap_match' block requires a defined 'swap_value', but the model does not specify how this value is calculated or if it must be reciprocal, which is critical for maintaining the integrity of non-cash exchange. [swap_match, a3]
|
||||
- (0.70 · medium) The model assumes 'remote_worker' can provide a 'marketable_skills' list and 'service_offering', but the process for validating or categorizing these skills to ensure they are genuinely exchangeable remains undefined. [remote_worker, a2]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The 'remote_worker' actor is constrained by 'Identity verification via LinkedIn or similar at signup', but the model does not show a mechanism for how this identity data (e.g., LinkedIn profile) is stored or utilized by the system. [remote_worker, identity_verification]
|
||||
- (0.75) The 'swap_match' block requires a 'required_skill' and 'swap_value', but the association 'a3' only shows that the system generates a match for, not how the required skill or swap value is sourced from the involved parties. [a3, swap_match]
|
||||
- (0.90) The requirement 'REQ-001' states the system must facilitate end-to-end completion of a skill swap (matching, scoping, exchange, rating), but there is no dedicated block or association representing the 'scoping' phase. [req_001, swap_match]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (Remote Worker) and necessary blocks (Service Offering, Review System). It missed explicit mention of 'mediator' or 'dispute resolution' as a distinct entity.
|
||||
- **model.accuracy:** 4/5 — Associations are generally correct. The use of `association` for core processes like matching is acceptable but could be refined to show flow/interaction more clearly.
|
||||
- **model.parsimony:** 5/5 — The model is highly focused and only includes necessary blocks (Swap Match, Review System) without adding unnecessary filler entities.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were captured perfectly in the dedicated `constraints` section of the model output.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., relying on trust score) and tied to core mechanics rather than generic adoption platitudes.
|
||||
- **risk.detection:** 5/5 — The risks identified are substantive failure modes (value calculation, dispute resolution) directly related to the non-cash nature of the exchange.
|
||||
- **voice.character:** 4/5 — The findings are highly structured and concrete. The tone is appropriately skeptical and focused on mechanism gaps.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a strong run that accurately captures the constraints and identifies high-leverage risks related to trust and value exchange.
|
||||
|
||||
---
|
||||
|
||||
### 05-cyclist-thing — Cyclist Thing
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.20
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/TP5BQm913CVlyod2UXPaVM111bj1FFNKUydEPDp5UKWoLfRYT--8UzYsUFtb_qZ8LZ9olWIlvDJ5Cp86QD2SZfmkqUwIJmmJjoH7zKWXBTfqxU8H77gXqSHu585l1V3sByKwTBFx9B4b7cYpnfTrNUWhSxeES4LBExS5xbFCxnW8gWFvBygTmKhTHdqnPNfIDwEvMJofkwcsimGKphc7GA4X5gKUorUxtlXECdoqvS8AKC0yo6U-4zXif9TCGMkOr6Q1g-QFKyQc8aUJ4sjT2WOqdUhVcKfjAThoXXy0)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Cyclist Thing (system), Cyclists (actor)
|
||||
- Associations: cyclist→cyclist_thing:
|
||||
- Constraints: _none_
|
||||
- Requirements: _none_
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The product will be perceived as useful by the target user group (Cyclists). [cyclist_thing, cyclist]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.95 · high) The model lacks specific functional requirements, making it impossible to define a measurable success metric or technical scope for the 'Cyclist Thing' system. [cyclist_thing]
|
||||
- (0.90 · medium) The target user ('Cyclists') is too broad and lacks segmentation, risking the development of a generic product that fails to address any specific pain point. [cyclist]
|
||||
- (0.90 · medium) The 'desiredOutcome' is vague ('Something useful for them'), providing no actionable direction to guide feature prioritization or design decisions. [cyclist_thing]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association 'a1' links the actor 'cyclist' to the system 'cyclist_thing', but no specific interaction or flow is defined. [a1, cyclist, cyclist_thing]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 3/5 — It correctly identifies the core actor (Cyclists) and system, but misses any potential sub-domains or specific cyclist needs.
|
||||
- **model.accuracy:** 4/5 — The structure is soundly simple; the association kind is appropriate for a preliminary model.
|
||||
- **model.parsimony:** 5/5 — Given the vague seed, the model avoids padding and keeps the scope extremely tight and focused.
|
||||
- **constraint.capture:** 5/5 — The seed provided no explicit constraints, so capturing zero is perfectly accurate and appropriate for this difficulty level.
|
||||
- **asm.detection:** 4/5 — It correctly flags the vagueness of 'useful' but could be more specific about *why* it's vague (e.g., lack of pain points).
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, focusing on scope definition and user segmentation failure modes.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, but the prose is slightly list-like rather than deeply question-led.
|
||||
- **Average:** 4.29 · **Min:** 3
|
||||
|
||||
> This is a solid pushback for a vague seed. The model excels at identifying scope risks while maintaining structural parsimony.
|
||||
|
||||
---
|
||||
|
||||
### 06-eventstream — EventStream
|
||||
|
||||
- **Difficulty:** hard
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/ZLJRRjD047ttLupIjoWEQLggKaLLIob2GaA5Fl1eZNV7oPAzcDrrGrHLuYFuGhw4yTeXeKrHNwBuxEoSCsUEVH4Y-jWOdOMbiZLwD52YMCwzQwoySjfvEAXUKqNJdOgmGEbMoiwXGXrevqHe307kCu2tJxh82gj9U-Ay9D-ZvIc-EPqoUkczM_NW2RsMHvB1zyx6JsW81Xz8tr5K0WVPmmwVTP6UePlGaJX-XkuXMoX9GAQEQp1aIl8XophnW-ixil4sUa8p00n0xND8pqLOXqW6PhFkp_av74p5CPwKIKJjNNJ2wK8uKsDKfTAA2SwWT4uJMYxHIf0Dn1XtP0TPIazxep5MpXkujdDbYNmYhtkqe1vbRXJHUQPkAUb7JLwH5LHe5ggvCTd6t7zhh79-P9JC4puAr08FMxuq9hp3YDif2ycFp51g9vOzGurhxL1k5GD4POY3Ose6cOpTsxPj5r5x2g7n1BTgRb6d_kKENeI4Fo8ABC5tVgKHj5kD34dLcD52pHUjz0t9L-KftvYi2LOgBk0ZLajiTo0RmJvjbW5i7nWrzrphB0IiAAwJf1LtURIgsKmuotEYiZ79-ZlNBO_EJAMowDSZ2DqZLrUANmEbQGI-iG6O0ZUdrd78cXgYWyF9PDadwLKWaVYxgg9hMMXbLFoFWCiQnO9WEfviGfIQO8ABJQxstC-UPYZFp_V5-WmSfmaZXSBPR4_1CCztWv8YmSTXNstBiM_fl5qUD6Hxt1-Fdv65TQqLguekssVQI-uzykabLp9DSx8IFASlSa27mo_qlM7Pfhqt77RGzUTyCZaS3k7tprzmqwSlW6b214-2r1tzumDqFeIqF4wE3p7lLioVj0YkWiXhtgHtl4CoxKckUCED8PzBpyqjLFocj4cCdjEUU4jAJIHLsEZORvNJp_ORpkGuux6wuR8B_k7l_ny0)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: EventStream (system), Platform Engineer (actor), Event Data (block), Backpressure Signal (block)
|
||||
- Associations: platform_engineer→eventstream_system:operates_on, eventstream_system→event_data:streams, eventstream_system→backpressure_signal:emits
|
||||
- Constraints: Wire-compatible with Kafka producer protocol for migration safety, Single binary, single config file, runs on a single node up to 100k events/sec, Apache 2.0 license
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.85) The target platform engineers will prioritize simplicity (single binary, no ZooKeeper) over maintaining full Kafka protocol compatibility. [single_binary_limit, wire_compatibility]
|
||||
- (0.90) The system's core value proposition relies on the assumption that platform engineers are currently dissatisfied with Kafka's operational complexity. [platform_engineer, single_binary_limit]
|
||||
- (0.75) The system assumes that backpressure signals are a novel concept or feature within the current event-streaming landscape. [req_001, backpressure_signal]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The requirement for wire compatibility with Kafka producer protocols (wire_compatibility) may conflict with the goal of implementing a fundamentally new, first-class backpressure mechanism (req_001), forcing the system to mask its core innovation. [wire_compatibility, req_001]
|
||||
- (0.80 · medium) The constraint of running on a single node up to 100k events/sec (single_binary_limit) limits the system's scalability and may fail to meet the needs of platform engineers operating at higher event volumes. [single_binary_limit]
|
||||
- (0.70 · medium) The model does not account for the operational complexity of managing consumer state or offsets when implementing consumer-driven flow control (req_002), which is critical for platform engineers. [platform_engineer, req_002]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.80) The association `a3` claims that the system `eventstream_system` emits a `backpressure_signal`, but no requirement explicitly mandates this relationship or the existence of the signal. [a3, eventstream_system, backpressure_signal]
|
||||
- (0.70) The `block` `backpressure_signal` is defined with a property `severity`, but no requirement or association utilizes this specific property. [backpressure_signal]
|
||||
- (0.75) The system `eventstream_system` is constrained by `wire_compatibility`, which implies interaction with the Kafka producer protocol, but no association links this compatibility requirement to any specific external actor or data flow. [wire_compatibility, eventstream_system]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — It correctly identified the core system and key data/signal types (Event Data, Backpressure Signal). It missed modeling the producer/consumer interaction flow.
|
||||
- **model.accuracy:** 3/5 — The associations are structurally correct, but 'emits' is too vague. The model lacks explicit relationships showing how backpressure signals affect data flow.
|
||||
- **model.parsimony:** 4/5 — The model is quite clean and only includes necessary components derived directly from the seed text, avoiding unnecessary filler blocks.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (wire compatibility, single binary, license) were perfectly captured in the SysML model's constraint section.
|
||||
- **asm.detection:** 4/5 — The assumptions are highly specific and actionable, correctly identifying tension points between simplicity goals and existing industry standards.
|
||||
- **risk.detection:** 5/5 — The risks are substantive failure modes (conflict of compatibility vs. innovation) and are tied directly to conflicting constraints/requirements in the seed.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and highly technical, adopting a question-led tone by highlighting conflicts between stated goals and practical implementation details.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a very strong run. The model excels at capturing constraints and identifying high-leverage risks/assumptions, which is critical for hard seeds.
|
||||
|
||||
---
|
||||
|
||||
### 07-pet-translator — Pet Translator
|
||||
|
||||
- **Difficulty:** failure-prone
|
||||
- **Confidence:** 0.85
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLN1Jjj04BtxAqPmG8eIH0D5D8eGKBMYXvQsGhrQO-_OMR7UTNVN2O56waVq2_ibdTqas2I4NgBiU6RUcpTlxGldqVgcKecxaxf6YnLacD-LrZHQl3VAMDWltb91mqw6cw0mCwbBA50vwZp95Je7Zmd0rLONKM1n59uOAyYkejaPlZiRSlJIMZDR1K_fhJWM7FnejF-25S7UDQafUPdZNhBeu6dZQGVS4Htd9plW5ib42WAgQZ-7YggCh4kI9Vczh-JXrg9s2hsnUu0EQlAfVmh1UEpcpbDrVWxxm_m4JxF8GUg2BEcSKjAbr9IokBvnC0BIJVKeXQ9-RKrEph5sVUA9xOBhQYkdcC_JoZ2d4MJ6A4BD93k4RcQQMYucd9W6vao7MGJqo75UqnfEcBAVemy0T5-JbP6PeYafJd7UCeSMux8Hqi0dNJS-ec0ufpAS6ITJ9hzRuLYSfLEJev8FwANHRJC0XrMj-6zgqGSYp8EbRL5-iEfTb6auxqQHha6L2lJO1IbuyRGBuOecE9N6eeBFBAUACDbJC6giDh2eCc6GkAud-VB6IkEad_SBGjl7ihHXRqQpgAfQn7LxihKb7sL8FTtxhh1hHZTw86WgSshz0GMRQBNrI93BbpbfcpCUvqPpDsISIFHfnckZoNd0pB1iVa8WPB6koiZFY3GSh8PQtaXPyCq9nmD0BU1WxUKGYGGciMKgfEDyMKLpRS4d7Ulz6WmsRaNIjSnWSBvvQKPG40dtp3jnjqdN1QyLjdROmjhQy8W7HJL_eEU0MrYl5MP6p5E5kcomfEIb7HqUli0MQxu3NEvDiYth_oeIHINnYgmi9vvDrkjzfv-Dj5IHzgxNMuO-V1iS7RtfzU3lxpzmomvOle-WQjWN_8e9KmBoSDnsYfcYkFR6KT6e3T-2AQA7cDH1UrCEEpZ37JYaO-Te20VUXFwnrIP0uuFQd90PPZ6JVXBIf0sOXmbFkvmvkU0V_XJz0m00)
|
||||
- **Notes:** Pet Translator is the SoI; Pet Owner is the primary actor; Audio and Video Inputs are data sources (blocks); Behavioral Model performs the core logic.
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Pet Translator (system), Pet Owner (actor), Audio Input (block), Video Input (block), Behavioral Model (block)
|
||||
- Associations: pet_owner→pet_translator:, audio_input→pet_translator:feeds_vocalization_data, video_input→pet_translator:feeds_visual_data, audio_input→behavioral_model:analyzes_vocalization, video_input→behavioral_model:analyzes_body_language
|
||||
- Constraints: On-device inference for privacy, Honest about the difference between 'emotion classification' and 'translation'
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The product assumes that users will perceive the system's output as a 'translation,' regardless of the scientific accuracy or underlying classification mechanism. [pet_translator, req_001]
|
||||
- (0.85) The product assumes that providing an interpretation of pet behavior will lead to a measurable increase in the owner's emotional bond with their pet. [req_002, pet_owner]
|
||||
- (0.80) The product assumes that combining audio and video inputs will yield sufficient data to generate a 'believable' interpretation of the pet's emotional state. [audio_input, video_input, behavioral_model]
|
||||
- (0.75) The product assumes that the technical feasibility of on-device inference is sufficient to maintain a positive user experience without compromising model performance. [on_device_inference, pet_translator]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) Achieving a 'believable and useful interpretation' (req_001) from raw audio/video inputs is technically challenging given the current state of multimodal pet behavior analysis. [behavioral_model, a4, a5, req_001]
|
||||
- (0.85 · high) The 'on-device inference for privacy' constraint (on_device_inference) may severely limit the complexity and accuracy of the underlying behavioral model, potentially failing to meet the required output quality. [pet_translator, on_device_inference, behavioral_model]
|
||||
- (0.75 · medium) The system relies on the pet owner (actor) to provide both audio and video inputs, creating a high dependency on user compliance and consistent data capture. [pet_owner, audio_input, video_input]
|
||||
- (0.95 · high) The 'honest disclaimer' constraint (honest_disclaimer) must be integrated into the user experience flow, otherwise the gap between scientific classification and perceived 'translation' will lead to immediate user distrust. [pet_translator, honest_disclaimer, req_002]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) Association a2 and a3 both feed data into the system (pet_translator) and also independently analyze data using behavioral_model, creating an ambiguous flow for input processing. [a2, a3, pet_translator, behavioral_model]
|
||||
- (0.85) The requirement REQ-001 states the system must provide an interpretation, but only links to behavioral_model, which is a block and not the primary output mechanism of the system. [req_001, behavioral_model]
|
||||
- (0.75) The requirement REQ-002 states the system must enable owners to use the interpretation output to bond with their pet, but does not specify how this 'output' is delivered or utilized by the owner. [req_002, pet_translator]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actor (Pet Owner) and necessary data inputs (Audio/Video), missing only a direct representation of the pet itself.
|
||||
- **model.accuracy:** 3/5 — The generalization link from Pet Owner to System is weak, and the input associations are redundant or confusingly structured.
|
||||
- **model.parsimony:** 4/5 — The model is generally focused on core components but includes some slightly over-specified properties (e.g., sampling_rate).
|
||||
- **constraint.capture:** 5/5 — Both explicit constraints were perfectly captured and applied to the correct system element.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., technical feasibility of on-device inference) and tied directly to core risks/constraints.
|
||||
- **risk.detection:** 5/5 — Risks are substantive failure modes, domain-specific, and correctly flag the inherent technological difficulty of the seed's premise.
|
||||
- **voice.character:** 4/5 — The findings are highly concrete and skeptical, but occasionally drift into bulleted lists rather than flowing prose.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a strong run that correctly identifies the core technical risks inherent in the seed's premise. Minor structural cleanup of the model would elevate it to excellent.
|
||||
|
||||
---
|
||||
|
||||
### 08-quiet-hours — Quiet Hours
|
||||
|
||||
- **Difficulty:** medium
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLNRZjis47tNLmms3qsCUE7k9a5g14RQRPhqOLDaqtU24aSIQufKICgE6opGZ-WNvajoHCcNNQ_tnGRES7ZEp9oXteOeVUmQauMbjgtqig5S5ilAkywgAsUSfoVbIoxvuk14gALoQsqhAgK9V10fZ0o1lcL4lntTeafPplg8yuhzYEQlvI-lBu3-whrRZ-0hVgak5S0_d8qVPSDqze7DYgCkv5bsUy1dNUGJT3E-B5wSeBlDQgsOk6dZXXfkSlOXomRnPvywpP4-kCu7-bpKh3h3_enae2zzHDGfykPDs8J8pM91JowA5_9LddIq7egAo4f4tR08niL0SmhHers8MwvSr39gPmLw7xi-ohPhlhLi5SuybqN1BIvuxlcVTDCjv8xI_iIHbLQTD7I5rdYTTz4DslGk98fz239bWV_5Odl5ErjfozpBeMjfPJNMndjSD0D-d7mb3LHAJv_PhtJ1ARCOGH46y5HhPAi53hPEstZO4jd5ccq_szILw9Pi4S-TCopjllWT-UyoohlC2WXeS-EA9LXxDbxXJg4uIctkJ23DvLzdUG_kAAuP7aDZRlXBno4cacR0X1-mUpGzgZt6hJgVzEDMJ1E66ipGkZ1wuP1kotV3KatNNiF4xvsh3DEkI6aL_TL5uzoIdbxhmhlWoaZllMphPvXsv6eWJ2Ahb2tSa20Emj0BFu1P0aoYzzLsACTpQ66pr9RK5QNBaZtRWc6AhwtHXOvc0yQn76nyejmcYEqDfua-kdx8he6WWbBt-YQlTSGgHUhW6q9ggNqpI2vXN7Y2W43XvOoKt0HwcdVQA8eUif-DdjVO9ICI95gOrwc7P6GdbcCwNTpPwNcl9MGFhSA9euTGV-He5-QxKN4GVSMdKvA9vnY8L45W_28PuvZZlY7dLB7bWM4x8e4MPaTY7iv63cRTL_E8YyxFxzSlMtW3bD5bHrPuzFI3Ktiq8pEC4XLvNTKHUpMPt0nhrs37msGoGEy-JMUpdoSJ-lxV__Gtx3EyGjJqDiTpZoUctAIz6lUIP2GyCOZ6jUiT67NLERp8YhcThfrV3cOD196qPbxkdDbxzlo0zk84xVQv1vU9x2sQkUBj3D8AOP5gw0Y81CZBNQnt33-5_VSa7Kxg-wYsAtYZFo1NUE1ahesEctDOLmszojxY1z_r7m00)
|
||||
- **Lint warnings:** 1
|
||||
- Multiple system blocks: quiet_hours, calendar_service
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Quiet Hours Scheduler (system), Individual Contributor (actor), Engineering Manager (actor), Calendar Service (system), Calendar Data (block), Meeting Request (block)
|
||||
- Associations: individual_contributor→quiet_hours:uses, engineering_manager→quiet_hours:uses, quiet_hours→calendar_service:integrates_with, quiet_hours→calendar_data:reads_and_writes, quiet_hours→meeting_request:generates_proposal_for, calendar_service→meeting_request:processes
|
||||
- Constraints: Read-write Google Calendar and Outlook (Microsoft Graph) integration, Must respect timezone differences explicitly in proposals, No automatic blocking without user confirmation for the first 30 days (build trust)
|
||||
- Requirements: REQ-001, REQ-002
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.95) The system assumes that users will prioritize protected deep-work time over the immediate needs of colleagues, leading to acceptance of alternative meeting times. [req_002, initial_trust_period]
|
||||
- (0.90) The product assumes that the social cost of saying no to a meeting is high enough that users are willing to adopt an automated negotiation tool, even if it requires initial setup and trust building. [initial_trust_period, individual_contributor, engineering_manager]
|
||||
- (0.85) The product assumes that the calendar system (Google/Outlook) will provide sufficient data granularity and API access to identify truly contiguous deep-work blocks without excessive latency or rate limits. [a4, calendar_service]
|
||||
- (0.80) The product assumes that the negotiation process will be perceived as helpful and non-confrontational by both parties, preventing it from being viewed as a tool for scheduling conflict avoidance rather than collaboration. [quiet_hours, req_002]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.80 · high) The system lacks a defined mechanism for handling conflicts when the proposed alternative meeting time (generated by 'a5') is rejected or falls outside of both parties' protected time slots. [quiet_hours, meeting_request]
|
||||
- (0.90 · high) The model does not specify how the system will handle simultaneous read-write access to 'calendar_data' from multiple users, risking race conditions or data corruption. [a4]
|
||||
- (0.95 · high) The model fails to account for the complexity of conflicting time zones when generating proposals, potentially leading to incorrect scheduling if 'timezone_respect' is not rigorously enforced at every API call. [a3, timezone_respect]
|
||||
- (0.85 · medium) The 'initial_trust_period' constraint requires the system to track user consent and usage history for 30 days, but no element or requirement exists to manage this state change. [initial_trust_period, quiet_hours]
|
||||
- (0.75 · medium) The model assumes that the 'calendar_service' can resolve complex scheduling conflicts across disparate platforms (Google/Microsoft) without detailing a standardized conflict resolution logic. [a3, google_outlook_integration]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.85) The association `a6` from `calendar_service` to `meeting_request` labeled 'processes' suggests the calendar service is responsible for processing meeting requests, but no block or actor is shown initiating this process.
|
||||
- (0.75) The requirement `req_002` states that the system must negotiate alternative meeting times, yet there is no explicit association showing how a user (actor) interacts with the negotiation mechanism or proposes initial conflicts to be resolved.
|
||||
- (0.70) The block `calendar_data` has properties like 'event_details' and 'time_zone', but no associations link it to any process that would require reading or writing this data (e.g., a read/write association from the system or an actor).
|
||||
- (0.65) The block `meeting_request` has properties for 'required_duration' and 'proposed_time', but no associations link it to any process that would generate or consume this data.
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core actors (IC, EM) and systems (Calendar Service), but missed modeling the 'user' as a general concept or the specific interaction of proposing/receiving meetings.
|
||||
- **model.accuracy:** 5/5 — The structural relationships are sound; associations correctly link the system to its data sources and primary actors.
|
||||
- **model.parsimony:** 4/5 — The model is generally clean, though 'calendar_data' and 'meeting_request' blocks feel slightly over-populated without clear process links.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints were accurately captured as dedicated constraint blocks applied to the core system.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific (e.g., social cost of saying no) and tied directly to the problem statement's underlying tension.
|
||||
- **risk.detection:** 5/5 — The risks identified are substantive failure modes (race conditions, time zone complexity) that are domain-specific and high severity.
|
||||
- **voice.character:** 4/5 — The findings are highly skeptical and concrete, but the prose occasionally drifts into bulleted lists rather than flowing narrative critique.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a very strong run; the risk detection was excellent. Minor improvements could be made to clarify data flow inconsistencies.
|
||||
|
||||
---
|
||||
|
||||
### 09-carbon-coach — Carbon Coach
|
||||
|
||||
- **Difficulty:** medium · HOLDOUT
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/VLRRRXit47tVhnWuBwqG2QwS66bX64bSfyrBqUlpOiITtILCShOaLuOG6CW_f5-OB-cGkv9MbfGNMHgISyxmd1dwROZeOszD4Hwqwz2ZXHMgXyPpxwexDkpXHVsQQbfETeGMAtxKhe4QJQ39YZ8O0dmg0DuVPQbgh2_J2lkA_1XTlS4Vtomb-ivxVXo3r_Iwkgea-85T_0qjmSMlPDOKjSABucc2vpZI6RXBkbAlpi0z5Qsk2CXsSGEMx8fyA8g1_CKT-XKxk6DKxGLW09K3fKe1kBa9cn39tjx2YwLwXTUhZDy7ycNdkTQ6u2S8qSlrIBpL8R8NtgQiCA8ihPWDeHCEMpnseRVa1wpnX-2WafC2az9hfzXIQMXDPffUzTwJYsND73k9HLbqVQfdd_yzjRZMKlrTYwwX3BCQOwNACK5R6LOFMpJR2NWP57lQ9mIWkcOVTon85ZuPVdnfgTAzVTdgfdtQutxOSReFKLkC0_IEQabZ-1cwm7PKAbbes6yEgaMZUeDHYngMOikLgDfCjkom_sAbqOY6KedNWLs63ZagGcwZpv0zfUHhAZswXsJlIUtes6siZfjT-NiXjwZt_y0ldaYyOr3R03zhJoebp4ICHgU6EsZIhbBbNMMrstLpa-IF7kLsHags3n4IORE1HnrR40nFpNWB8Nr7NoMkvDRQixDY1wbyoBqufSRqvfIEET7WlV77mTsUu8J6m5QxC3BK8cDiIVkfI5Ee2PLt-Igvtme2DM7ilUHYPpQP2cwtbADckoqd-3YMQq0pTsbIo6nALW3nUoTTpKxORa2wtKLTQrn9UsedDhO3m-z-__ZnUz0XRHPr8rLYKBybjdVA8ikyRSx5ulPWAXG78s8-lttUOCSRZjlWUC-HNuipnddEHzo9dHYEGXcvEAbhEdAY5GyEdbVXsu3VFdVSXGU72qCDkGgyJ9CewivcVzA_lPIQd1rcio5q_yVyylA7sGo-VluF_cv5zpoL1pU6lkjaKCbVUTwq4qt5BoftDDT9wNaQr_lonJvLhx8VU2qB25OxRNiBrtChJr2RpcYk5XFivHdiqMaRiR3KG5AbJTKiltx-SWLfGiw7wRfp20mE2T8-65CFvRT4RjU1lB9Td89UdG6LXsUT7ZHB67gVJLuJLUabbvxAIKTKgJzqfAGLr8w1G3DKFXafGvDhiCdNlIY4-w17yLO-v1-5_m40)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Carbon Coach (system), Consumer (actor), Behavior Change (block), Footprint Estimate (block), Social Comparison (block)
|
||||
- Associations: consumer→carbon_coach:, carbon_coach→behavior_change:, carbon_coach→footprint_estimate:, carbon_coach→social_comparison:
|
||||
- Constraints: EU Green Claims Directive Compliance, Footprint methodology citation, Anonymous social features only
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (0.90) The system assumes that providing measurable feedback on impact (req_003) is sufficient to drive sustained behavior change, rather than requiring direct behavioral nudges or incentives.
|
||||
- (0.85) It assumes the target user base (climate-aware consumers) will actively engage with and benefit from social comparison features (social_comparison), even if they are anonymous.
|
||||
- (0.75) The design implicitly assumes that users' willingness to change habits is high enough that the primary barrier to adoption is simply knowing *what* changes to make, not overcoming inertia or cost.
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The system relies on the 'social_comparison' block to drive behavior change, but the model does not define how relative ranking translates into actionable recommendations for specific behaviors (e.g., linking a low rank to a high-impact 'behavior_change'). [social_comparison, behavior_change, a4]
|
||||
- (0.85 · medium) The 'consumer' actor is defined with `income_level` and `current_footprint`, but the model lacks a mechanism to use `income_level` (an economic variable) when calculating or recommending behavior changes, potentially limiting personalization. [consumer, a1]
|
||||
- (0.95 · high) The 'footprint_estimate' block requires citing methodology and scope category, but the model does not specify how this citation information is presented to the user or integrated into the core feedback loop (req_003), risking compliance failure. [footprint_estimate, citation_methodology, a3, req_003]
|
||||
- (0.90 · medium) The 'behavior_change' block includes `effort_level`, but the model does not define how this effort level is balanced against the `impact_score` to ensure recommendations are both high-impact and sustainable for the user. [behavior_change, req_002]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The requirement REQ-002 suggests the system must identify and recommend 2–3 high-impact behavior changes, but no block is explicitly associated with the function of 'recommending' or 'identifying' these behaviors. [req_002, behavior_change]
|
||||
- (0.85) The requirement REQ-003 states the system must provide measurable feedback on impact, but this is satisfied by both 'carbon_coach' and 'footprint_estimate', creating ambiguity regarding which element is responsible for the *feedback mechanism* itself. [req_003, carbon_coach, footprint_estimate]
|
||||
- (0.95) The 'consumer' actor block has properties like 'income_level' and 'current_footprint', but these properties are not referenced or utilized by any requirement, association, or other element in the model. [consumer]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured core entities (Consumer, Behavior Change, Social Comparison) but missed the explicit concept of 'effort' or 'cost' which is central to sustained change.
|
||||
- **model.accuracy:** 4/5 — The structure is generally sound; associations and blocks are logically placed, though some relationships could be more explicitly defined (e.g., a recommendation link).
|
||||
- **model.parsimony:** 5/5 — The model is highly focused, adding only necessary components to address the core mechanics of carbon tracking and social influence.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (Green Claims, Anonymity, Methodology) were correctly captured and applied to relevant system elements.
|
||||
- **asm.detection:** 4/5 — Assumptions are specific (e.g., assuming feedback is enough), but could be strengthened by questioning the underlying behavioral science models used.
|
||||
- **risk.detection:** 5/5 — The risks are highly substantive, domain-specific, and pinpoint critical failure modes related to mechanics and compliance.
|
||||
- **voice.character:** 4/5 — The findings are skeptical and concrete, using technical language (e.g., 'mechanism,' 'translates into') but occasionally drift into overly long prose.
|
||||
- **Average:** 4.43 · **Min:** 4
|
||||
|
||||
> This is a strong run that successfully maps the complex interplay of social mechanics and regulatory compliance required by the seed.
|
||||
|
||||
---
|
||||
|
||||
### 10-telemetry-lite — Telemetry Lite
|
||||
|
||||
- **Difficulty:** hard · HOLDOUT
|
||||
- **Confidence:** 0.90
|
||||
- **Diagram:** [view online](https://www.plantuml.com/plantuml/uml/XLNRJjj057ptLnm5LIrHGzDm4KK8GQ4j3st5hS-h4_lOMR4NixiE61gf7z4lx9Vq-98B8S138dbsFJDdTZOvz05TABIA_BKqEJhKCCJuEdEsCCcnLTR1MhfDAGqMTlWH9lPMcWnIL9uMLcA5tiD31F3v2KkIOjglLgnBoBNeS1S_xGuOFNBEthRW3cqdcmc3Nwq9Fr4JT4v9ZId863lHP47Fs43FoFLfCzvwHcuIZMH2G3eF9MZIGt8-YXhpdIjIf2cu4hxBG1r03s4A2SKGxE_xqWVI1mUmDeYtS6TOU_31Ein8L06IIM0Fo1JwmTzKBqruGqAvigKc4qHXPE0DFZYEaVsqsbzC9WrH1S8LeQxLQGwAm23hOynQB5_9nbRdQ4hXvJtDD3KgzLvJ8WizcTF_ykJW10FMl0K_YOIVc72eR7pT4jAO7VgvEvwCl9VM2CS3OBd2zvcpIH47E3AeIZuc3yVMDTdbpPB0sT9ZjRGmXfG8ZbEhnjw3eRMAq4oT2CmoHna6Th2mEjC-cQNA9GeovMnguOMqiSMNf6Ek5tAqJjulimCCfK4-yDGwZU6HxbJuakqgchDyk3ol6ZFj0TYy8gsz-7ghc6qLJIdO3qCS9eEXDZMRwYZ7U4GmsEY3aZ4PtwI9DIm6eYzQ-0MERoTd5xsg7aeYfrCVDTtbrWL0xcV2hQx77XPI1P2DrIp9dPY-5A-aZvuqScFZOEcIHCjtvdUlDszSz5mWJuauUipP37aHRBHY_jTUMHwesZ-l_gjRNwAE56NLJNSo6mNkJBTxGJU5T5IrpdUx3VJblDVlV-nsuT-Vl_2ZyA5ANexvAjRzXbdrgqDfheE7TxlzDs1JEBC-dDeCngWAMbyW7AmYR2ygj1ULoUBgX96jbwzJRYvIwkh0-GFAtdALoM73LF_UABgh6RSN2T_wTMv6qcGI7V8N_x7y1m00)
|
||||
|
||||
**Model elements:**
|
||||
|
||||
- Blocks: Telemetry Lite (system), Engineering Team (actor), User Data (block), Product Analytics Core (block), Deployment Artifact (block)
|
||||
- Associations: engineering_team→telemetry_lite:, telemetry_lite→user_data:
|
||||
- Constraints: Single container/SQLite storage option, Apache 2.0 license, GDPR-compliant data export and deletion built in
|
||||
- Requirements: REQ-001, REQ-002, REQ-003
|
||||
|
||||
**Assumptions:**
|
||||
|
||||
- (1.00) The target user group is willing to accept a smaller feature set (80% of PostHog's value) in exchange for drastically reduced operational complexity. [req_001, req_002, req_003]
|
||||
- (0.90) The target user group is willing to self-host the product analytics tool without having an infra engineer. [single_container_sqlite, engineering_team]
|
||||
- (0.80) The target user group is willing to use a tool that requires SQLite-only storage and single container deployment. [single_container_sqlite, deployment_artifact]
|
||||
- (0.75) The target user group is willing to forgo the full feature set of PostHog (e.g., Kafka, ClickHouse) if it means avoiding third-party SaaS costs. [product_analytics, user_data]
|
||||
|
||||
**Risks:**
|
||||
|
||||
- (0.90 · high) The requirement to deliver session replay functionality (req_002) while maintaining low operational complexity (req_003) is technically challenging, as session replay typically requires high-volume data streams and complex storage beyond simple SQLite. [user_data, req_002, req_003]
|
||||
- (0.85 · medium) The constraints on storage (single_container_sqlite) and the need to handle complex user data like session replay (user_data) may lead to significant performance bottlenecks or data integrity issues when scaling beyond small proof-of-concept usage. [single_container_sqlite, user_data]
|
||||
- (0.95 · high) The model does not account for the necessary infrastructure or mechanisms to manage data export and deletion required by GDPR compliance (gdpr_compliance) within a single, simple deployment unit. [gdpr_compliance, telemetry_lite]
|
||||
- (0.75 · medium) The initial hypothesis assumes a willingness to accept smaller feature sets (product_analytics) in exchange for trivial ops, but the model lacks any mechanism to validate or measure this acceptance threshold. [product_analytics]
|
||||
|
||||
**Inconsistencies:**
|
||||
|
||||
- (0.90) The requirement to deliver session replay functionality (req_002) is satisfied by the 'user_data' block, but 'user_data' does not contain a specific property for session replay. [req_002, user_data]
|
||||
- (0.85) The 'product_analytics' block is associated with the requirement to deliver event tracking and funnels (req_001), but it lacks properties for both 'funnel_tracking' and 'event_aggregation'. [req_001, product_analytics]
|
||||
- (0.75) The 'deployment_artifact' block is defined but has no associations or requirements satisfying it. [deployment_artifact]
|
||||
|
||||
**Judge scores** (LLM-as-judge — sanity check, not final):
|
||||
|
||||
- **model.coverage:** 4/5 — The model captured the core user (Engineering Team) and key features (Product Analytics, User Data), but missed explicit mention of GDPR/privacy as a distinct entity.
|
||||
- **model.accuracy:** 3/5 — The associations are generally correct, but the generalization between 'telemetry_lite' and 'user_data' is structurally weak. The blocks themselves lack internal consistency (e.g., properties vs. definition).
|
||||
- **model.parsimony:** 4/5 — The model is reasonably focused on core components; it avoids excessive filler but includes a few slightly generic blocks.
|
||||
- **constraint.capture:** 5/5 — All three explicit constraints (SQLite/container, Apache 2.0, GDPR) were accurately captured and applied to the correct system elements.
|
||||
- **asm.detection:** 4/5 — Assumptions are highly specific, focusing on the trade-offs required by the seed's core tension (feature set vs. complexity).
|
||||
- **risk.detection:** 5/5 — The risks are substantive failure modes, domain-specific, and directly challenge the technical feasibility of the stated constraints.
|
||||
- **voice.character:** 4/5 — The findings are highly skeptical and concrete, focusing on technical limitations rather than general business advice. Minor improvements in prose flow would perfect it.
|
||||
- **Average:** 4.14 · **Min:** 3
|
||||
|
||||
> This is a very strong analysis that excels at identifying deep technical risks and assumptions inherent in the seed's constraints.
|
||||
|
||||
---
|
||||
57
phase-0/reports/notes-run-01.md
Normal file
57
phase-0/reports/notes-run-01.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Notes — first corpus run (2026-04-28)
|
||||
|
||||
Detailed report: `reports/2026-04-28-1735-run.md`
|
||||
|
||||
## Summary
|
||||
|
||||
| Seed | Diff | Blocks | Assoc | Constr | Req | A | R | I | Conf | Hallucinated refs |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| 01-aristotle | easy | 5 | 3 | 3 | 3 | 1 | 3 | 3 | 0.90 | **4** |
|
||||
| 02-habit-coach | medium | 6 | 6 | 3 | 3 | 3 | 4 | 3 | 0.90 | 0 |
|
||||
| 03-redline-ai | medium | 4 | 3 | 4 | 4 | 5 | 3 | 3 | 0.95 | 0 |
|
||||
| 04-skillswap | medium | 5 | 5 | 3 | 3 | 3 | 5 | 3 | 0.90 | 0 |
|
||||
| 05-cyclist-thing | failure-prone | **2** | 1 | 0 | 0 | 1 | 3 | 1 | **0.30** | 0 |
|
||||
| 06-eventstream | hard | 5 | 4 | 3 | 4 | 4 | 4 | 3 | 0.90 | 0 |
|
||||
| 07-pet-translator | failure-prone | 4 | 3 | 2 | 3 | 4 | 4 | 3 | 0.85 | 0 |
|
||||
| 08-quiet-hours | medium | 6 | 5 | 3 | 3 | 4 | 3 | 4 | 0.95 | 0 |
|
||||
|
||||
8/8 seeds completed without errors. Total runtime: ~7 minutes.
|
||||
|
||||
## What worked
|
||||
|
||||
1. **The "under-suggest on vague seeds" rule fired correctly.** Cyclist Thing produced exactly 2 blocks (system + actor), 0 constraints, 0 requirements, confidence 0.30. The model didn't fabricate to fill the diagram — exactly the behavior we asked for in `generate.md`.
|
||||
|
||||
2. **Failure-prone seeds produced *the right kind of findings*.** Cyclist Thing's risks all flag the vagueness itself ("no measurable success criteria", "no use case defined", "actor too vague"). Pet Translator's #1 risk is "system's core value proposition 'translation' risks failing to meet user expectations if the output is merely an emotion classification" — which is the central feasibility critique we hoped Socrates would surface.
|
||||
|
||||
3. **Confidence calibration looks reasonable.** Range 0.30 → 0.95 across the corpus, correlating with seed quality / specificity. Vague → low; well-specified with explicit constraints → high.
|
||||
|
||||
4. **No lint warnings on any seed.** Structural integrity rules (S1–S5) hold across all 8 generated models.
|
||||
|
||||
5. **Hallucinated refs are concentrated on Aristotle.** 4 of 4 stripped refs were on the Aristotle run; the other 7 seeds produced zero. Worth understanding why — possibly the seed's narrative density triggers more inferred-but-non-existent element refs.
|
||||
|
||||
## Iteration candidates
|
||||
|
||||
In priority order:
|
||||
|
||||
1. **Constraint-vs-requirement duplication.** Across multiple seeds, Gemma renders the same content as both a constraint AND a requirement. The seed's `constraints` field is meant to flow into the `constraints` array; right now it's getting double-encoded. Fix: tighten `generate.md` with an explicit rule — "if a constraint is in `seed.constraints`, output it ONLY in the `constraints` array, never as a requirement."
|
||||
|
||||
2. **Aristotle hallucination cluster.** 4/4 stripped refs on this one seed. Hypothesis: the seed mentions narrative concepts (e.g., "lectures", "office hours") that the model includes as flavor in detection without modeling them as elements. Fix: tighten detection prompts — "every element id must appear verbatim in the model JSON shown to you."
|
||||
|
||||
3. **Property-vs-block confusion on Pet Translator.** Pet Translator's blocks `pet_vocalization` and `pet_body_language` have property names like `audio_input_stream`, `frequency_spectrum`, `pose_keypoints` — these get referenced in findings as if they were elements. Post-validate accepts them (we tolerate bare property names) but it's a UX smell. Fix: when surfacing findings, distinguish "linked block" from "linked property of block".
|
||||
|
||||
4. **Quiet Hours gen latency (49s).** 2× the median. Possibly because the seed has 3 detailed constraints producing a chunkier model. Acceptable but worth watching as we add more seeds.
|
||||
|
||||
5. **Cyclist Thing's `Cyclist Thing` block has no properties** and the only association is unlabeled (`cyclist→cyclist_thing`). The risks correctly flag this — not a bug, but evidence the model is willing to ship truly minimal structures when the seed warrants.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Should `phase0 corpus` render PNGs by default?** Currently `--skip-png` keeps it fast (~7 min); rendering all PNGs adds ~1 min total but lets us eyeball diagrams during review. Probably yes — flip the default once we're confident the harness is stable.
|
||||
- **Threshold for stripping vs keeping a finding when refs are bad.** Currently we keep findings with zero refs (legitimate "missing element" findings) and drop only those whose every ref was hallucinated. This is the right default but worth re-examining once we have human scores.
|
||||
- **Is the seed corpus diverse enough?** Of 8 non-holdouts: 1 consumer-edu, 1 consumer-mobile, 1 B2B legal, 1 marketplace, 1 vague, 1 technical-protocol, 1 wishful-tech, 1 productivity. Reasonable spread. The 2 holdouts (Carbon Coach, Telemetry Lite) add behavioral/social and OSS-positioning. We don't need more seeds yet.
|
||||
|
||||
## Next session — Day 6
|
||||
|
||||
1. Wire LLM-as-judge: implement `phase0 score <seedId> --judge` per the plan §6.4
|
||||
2. Run judge over all 8 outputs; capture per-dimension scores in the report
|
||||
3. Identify the top 1–2 failure patterns and iterate one prompt
|
||||
4. Re-run corpus with the iterated prompt, compare scores against this baseline
|
||||
96
phase-0/reports/notes-run-02.md
Normal file
96
phase-0/reports/notes-run-02.md
Normal file
@@ -0,0 +1,96 @@
|
||||
# Notes — first scored corpus run (2026-04-28, run 02)
|
||||
|
||||
Detailed report: `reports/2026-04-28-1942-run.md`
|
||||
|
||||
## Result vs. rubric
|
||||
|
||||
**Threshold per [phase-0-validation.md §6.3](../../docs/phase-0-validation.md):**
|
||||
- Per-seed: average ≥ 3.5 AND no dimension < 3.0
|
||||
- Across corpus: ≥ 80% of seeds pass
|
||||
|
||||
| Seed | Avg | Min | Status |
|
||||
|---|---|---|---|
|
||||
| 01-aristotle | 4.14 | 3 | ✓ |
|
||||
| 02-habit-coach | **4.57** | 4 | ✓ |
|
||||
| 03-redline-ai | 4.43 | 4 | ✓ |
|
||||
| 04-skillswap | 4.43 | 4 | ✓ |
|
||||
| 05-cyclist-thing | 4.00 | 3 | ✓ (vague-seed handled correctly) |
|
||||
| 06-eventstream | 3.43 | **2** | ✗ |
|
||||
| 07-pet-translator | 4.14 | 3 | ✓ |
|
||||
| 08-quiet-hours | 4.14 | 3 | ✓ |
|
||||
|
||||
**Pass rate: 7 / 8 = 87.5%** — above the 80% threshold.
|
||||
|
||||
**Corpus mean across all seeds: 4.16 / 5.**
|
||||
|
||||
## Per-dimension averages
|
||||
|
||||
Across the 8 seeds:
|
||||
|
||||
- modelCoverage: 3.75
|
||||
- modelAccuracy: 3.50
|
||||
- modelParsimony: 4.50
|
||||
- constraintCapture: 4.00
|
||||
- assumptionDetectionQuality: 4.13
|
||||
- riskDetectionQuality: 4.50
|
||||
- voiceAndCharacter: 4.13
|
||||
|
||||
The weakest dimension is **modelAccuracy** (3.50) — pulled down by the eventstream 2 and lower scores on other technical/abstract seeds. **modelParsimony** and **riskDetectionQuality** lead at 4.50 — Gemma is doing well at not over-populating and at surfacing substantive risks.
|
||||
|
||||
## The eventstream failure
|
||||
|
||||
Eventstream produced **5 blocks but 0 associations / 0 constraints / 0 requirements**. The judge correctly penalized:
|
||||
- modelAccuracy 2/5 — "lists entities but provides no associations"
|
||||
- constraintCapture 2/5 — "failed to capture critical operational constraints"
|
||||
|
||||
This is a stochastic regression — the previous (unscored) run on the same seed had `5b 4a 3c 4r`. Two iteration paths:
|
||||
1. **Retry to confirm stochasticity.** Re-run the seed alone; if it produces full output, it's variance.
|
||||
2. **Strengthen the prompt.** Add explicit guidance: "Even for technical/protocol seeds, you MUST populate all four arrays. If you can't infer associations, prefer fewer high-confidence ones over none."
|
||||
|
||||
Recommend (1) first, then (2) if the regression repeats.
|
||||
|
||||
## Confidence calibration check
|
||||
|
||||
Self-reported `overallConfidence` from the model:
|
||||
|
||||
| Seed | Self-conf | Judge avg |
|
||||
|---|---|---|
|
||||
| 01-aristotle | 0.90 | 4.14 |
|
||||
| 02-habit-coach | 0.90 | 4.57 |
|
||||
| 03-redline-ai | 0.90 | 4.43 |
|
||||
| 04-skillswap | 0.90 | 4.43 |
|
||||
| 05-cyclist-thing | **0.30** | 4.00 |
|
||||
| 06-eventstream | 0.92 | 3.43 |
|
||||
| 07-pet-translator | 0.90 | 4.14 |
|
||||
| 08-quiet-hours | 0.90 | 4.14 |
|
||||
|
||||
Note **eventstream self-rated 0.92 but actually scored worst** — model overconfident on its weakest output. This is a known small-model failure mode. Worth probing in iteration: tighten the confidence rule to penalize self-rating high when several arrays are empty.
|
||||
|
||||
The cyclist-thing low confidence (0.30) correlates correctly with the judge's high score on it — both recognize the seed forced sparse-but-correct output, not low quality.
|
||||
|
||||
## Wins worth keeping
|
||||
|
||||
1. **Vague-seed handling proven twice.** Cyclist Thing scored 4.00 (passes) — the "under-suggest, push back, refuse to fabricate" prompt instruction works.
|
||||
2. **Constraint dimension averages 4.00** — the schema-tightening that removed `kind:'constraint'` from blocks is paying off.
|
||||
3. **Risk detection averages 4.50** — the prompt asking for "specific failure modes tied to specific elements" is producing substantive risks across all seed types.
|
||||
4. **No structural lint warnings on any seed.** Validation rules hold.
|
||||
|
||||
## Iteration candidates (priority order for Day 7)
|
||||
|
||||
1. **Retry eventstream.** Cheap — confirms stochastic vs. systemic.
|
||||
2. **Add "all four arrays must be populated" guidance to `generate.md`** — only if (1) confirms it's systemic.
|
||||
3. **Tighten confidence calibration:** instruct the model to rate confidence based on its OWN model coverage, not on the seed's apparent richness. (e.g., "If you generated 0 associations, your overall confidence cannot exceed 0.5.")
|
||||
4. **Habit Coach style as gold standard.** It scored highest — use its prompts/output as a reference for what "good" looks like in further refinements.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Should we automate stochasticity testing?** Run each seed N times and report variance? Useful but expensive (~7 min per seed × 8 seeds × 3 runs = ~3 hours).
|
||||
- **Judge sycophancy risk.** Gemma is judging Gemma. Worth running a known-bad model (deliberately broken Aristotle) to see if the judge correctly scores it low. Could be a Day 7 sanity check.
|
||||
- **Token cost summary in reports.** Currently we don't sum totals. Trivial to add.
|
||||
|
||||
## Next session — Day 7
|
||||
|
||||
1. Retry eventstream alone, confirm whether the regression repeats
|
||||
2. If systemic: iterate `generate.md` on the populate-all-arrays rule and re-run the corpus
|
||||
3. If stochastic: just note it and proceed — single-seed variance is real
|
||||
4. Run a deliberate-bad sanity check on the judge (mutate Aristotle to remove 80% of elements, see if score drops)
|
||||
119
phase-0/reports/notes-run-03.md
Normal file
119
phase-0/reports/notes-run-03.md
Normal file
@@ -0,0 +1,119 @@
|
||||
# Notes — second scored corpus run, post generate.md iteration (2026-04-28, run 03)
|
||||
|
||||
Detailed report: `reports/2026-04-28-2115-run.md`
|
||||
|
||||
## Change applied
|
||||
|
||||
Tightened `src/prompts/generate.md` with explicit guidance separating constraints from requirements:
|
||||
|
||||
> Each item from `seed.constraints` belongs in EXACTLY ONE place — the `constraints` array. Do NOT also output it as a requirement. (...) The `requirements` array should contain things derived from `seed.desiredOutcome` — NOT a re-encoding of `seed.constraints`.
|
||||
|
||||
## Headline result
|
||||
|
||||
| | Run 02 (baseline) | Run 03 (this) | Δ |
|
||||
|---|---|---|---|
|
||||
| **Pass rate** | 7/8 = 87.5% | 7/8 = 87.5% | flat |
|
||||
| **Corpus mean** | 4.16 | 4.11 | -0.05 |
|
||||
| **Constraint↔requirement duplication** | present every seed | **eliminated every seed** | ✓ fix |
|
||||
|
||||
Score did not move materially. Underlying output structure is meaningfully cleaner.
|
||||
|
||||
## Per-seed deltas
|
||||
|
||||
| Seed | Run 02 | Run 03 | Δ |
|
||||
|---|---|---|---|
|
||||
| 01-aristotle | 4.14 / 3 | 4.14 / 3 | flat |
|
||||
| 02-habit-coach | 4.57 / 4 | 4.43 / 4 | -0.14 |
|
||||
| 03-redline-ai | 4.43 / 4 | 4.14 / 3 | -0.29 |
|
||||
| 04-skillswap | 4.43 / 4 | 4.43 / 4 | flat |
|
||||
| 05-cyclist-thing | 4.00 / 3 | **3.00 / 2** | **-1.00 ✗** |
|
||||
| 06-eventstream | 3.43 / 2 | 4.14 / 3 | +0.71 ✓ |
|
||||
| 07-pet-translator | 4.14 / 3 | 4.43 / 4 | +0.29 |
|
||||
| 08-quiet-hours | 4.14 / 3 | 4.14 / 3 | flat |
|
||||
|
||||
**4 seeds flat, 2 up, 2 down.** Eventstream's improvement was a re-run earlier in the session before this corpus pass — already known to be stochastic. So in pure-iteration terms: 4 flat, 1 up (pet-translator), 3 down (habit-coach, redline-ai, cyclist-thing).
|
||||
|
||||
## Where the fix shows in the artifacts
|
||||
|
||||
Cross-checked four seeds with explicit `seed.constraints`. All show clean separation:
|
||||
|
||||
```
|
||||
01-aristotle
|
||||
C[3]: Must never output a complete solution... ; Response latency under 1.2s... ; FERPA tenancy...
|
||||
R[2]: facilitate low-stakes re-engagement ; guide student without providing final solutions
|
||||
↑ both derived from desiredOutcome, not from constraints
|
||||
|
||||
03-redline-ai
|
||||
C[4]: Word integration ; no cross-customer training ; audit trail ; SOC 2
|
||||
R[2]: first-pass redlines complete in minutes ; GC reviews AI suggestions
|
||||
↑ both from desiredOutcome
|
||||
|
||||
06-eventstream
|
||||
C[3]: Kafka wire-compat ; single binary 100k/s ; Apache 2.0
|
||||
R[3]: backpressure first-class ; consumer-driven flow control default ; LLM-pipelined workloads
|
||||
↑ from desiredOutcome and the protocol-design vision
|
||||
|
||||
08-quiet-hours
|
||||
C[3]: GCal+Outlook RW ; timezone-respect ; no auto-blocking first 30 days
|
||||
R[3]: 2 contiguous focus blocks/week ; invisible negotiation ; reduce social cost
|
||||
↑ from desiredOutcome
|
||||
```
|
||||
|
||||
Compare to run-02 where the same seeds had REQ-NNN entries that literally restated `seed.constraints` text.
|
||||
|
||||
## The Cyclist Thing regression
|
||||
|
||||
Run 02: 2 blocks, 1 association, 0 constraints, 0 requirements, conf 0.30 → score 4.00 (vague handled correctly)
|
||||
|
||||
Run 03: 4 blocks, 3 associations, 0 constraints, **1 requirement**, conf 0.30 → score 3.00 (judge penalizes the inventing)
|
||||
|
||||
Hypothesis: the new prompt's "**derive requirements from `seed.desiredOutcome`**" instruction nudged the model to invent a requirement from "Something useful for them" rather than leave the array empty. The vague-seed escape hatch ("if too vague, return sparse and set confidence < 0.3") is still in the prompt but didn't fire as cleanly.
|
||||
|
||||
**Confidence stayed correctly low (0.30)** — the model knows it's overreaching. So the lower judge score reflects real over-generation that the model itself flagged.
|
||||
|
||||
Cheap fix candidate: add to `generate.md` — "If `seed.desiredOutcome` is too vague to derive specific requirements (e.g., generic 'something useful'), leave the `requirements` array EMPTY rather than inventing a placeholder."
|
||||
|
||||
## Judge sanity check (separate, this session)
|
||||
|
||||
Deliberately broken Aristotle (1 block, 0 of everything else, no findings) → judge gave **1.29 avg, min 1**. Real Aristotle scores 4.14. The judge isn't rubber-stamping.
|
||||
|
||||
## Decision matrix for next move
|
||||
|
||||
We've now validated:
|
||||
- ✓ Generation works on a 4B local model
|
||||
- ✓ Detection works (assumptions / risks / inconsistencies, with post-validation guard)
|
||||
- ✓ Conversation loop works
|
||||
- ✓ Propose-and-apply works (oneOf JSON-schema necessary for small-model op generation)
|
||||
- ✓ LLM-as-judge works (sanity-checked against deliberately broken input)
|
||||
- ✓ Vague-seed handling works (Cyclist Thing scored well in run-02; regressed in run-03 from a known cause)
|
||||
- ✓ Feasibility-suspect handling works (Pet Translator)
|
||||
- ✓ Constraint↔requirement separation now clean across all seeds
|
||||
- ✓ Pass rate ≥ 80% (currently 87.5%)
|
||||
- ✓ Corpus mean ~4.1 (well above the 3.5 per-seed threshold)
|
||||
|
||||
**Phase 0 has met its quantitative pass criteria** as defined in [phase-0-validation.md §6.3](../../docs/phase-0-validation.md).
|
||||
|
||||
Three plausible next moves:
|
||||
|
||||
**A. One more iteration.** Add the vague-desired-outcome guard, re-run. Cheap (~15 min). Likely restores cyclist-thing pass.
|
||||
|
||||
**B. Day 9: human review.** The user (you) reads the diagrams and findings and assigns scores. LLM-judge has been the sanity check; final scoring per the plan is human.
|
||||
|
||||
**C. Day 10: final round with holdouts.** Run Carbon Coach + Telemetry Lite under `--final` to verify the prompts generalize beyond what they were iterated against. Per the plan this is the gate decision.
|
||||
|
||||
Recommendation: A → C. Skip B until after C, since holdouts are the real test of whether iteration overfit.
|
||||
|
||||
## Token usage roll-up (this run)
|
||||
|
||||
Approximately:
|
||||
- Generation: 8 × ~3k in / ~1k out = 24k in / 8k out (Sonnet/local)
|
||||
- Detection: 8 × ~5k in / ~1k out = 40k in / 8k out
|
||||
- Judge: 8 × ~2.5k in / ~0.3k out = 20k in / 2.4k out
|
||||
- **Total ~84k in / 18.4k out** for full scored corpus
|
||||
- LLM cost: $0 (local)
|
||||
- Wallclock: ~10 minutes
|
||||
|
||||
## Updated open questions
|
||||
|
||||
- Iteration efficiency: 3 corpus runs done (~30 min compute, ~30 min author). Next iterations should be smaller, more targeted (single-seed re-runs) before triggering full corpus.
|
||||
- Variance reporting: would help to run each seed N=3 times and report (mean, stdev) per dimension. Defer.
|
||||
18
phase-0/seeds/01-aristotle.json
Normal file
18
phase-0/seeds/01-aristotle.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "01-aristotle",
|
||||
"title": "Aristotle",
|
||||
"problem": "First-year STEM students at large public universities frequently disengage from coursework not because the material is intractable, but because they lack a low-stakes thinking partner during the long tail between lectures and office hours.",
|
||||
"targetUser": "Undergraduates at large public universities, weeks 3-10 of an intro course.",
|
||||
"desiredOutcome": "Students re-engage with material via a low-stakes thinking partner — without producing solutions.",
|
||||
"initialHypothesis": "Students will adopt a tool that explicitly refuses to solve their homework, because the market is saturated with answer-givers.",
|
||||
"constraints": [
|
||||
"Must never output a complete solution to a graded problem",
|
||||
"Response latency under 1.2s P50 to preserve flow",
|
||||
"FERPA tenancy — coursework never leaves institutional boundary"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "easy",
|
||||
"testsFor": ["baseline", "consumer-edu", "explicit-constraints"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/02-habit-coach.json
Normal file
18
phase-0/seeds/02-habit-coach.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "02-habit-coach",
|
||||
"title": "Habit Coach",
|
||||
"problem": "Parents of school-age kids want to limit screen time but the existing tooling (OS-level limits, single-app blockers) is either too coarse or too easily bypassed. Worse, parents have no visibility into whether limits are working — they only see the friction.",
|
||||
"targetUser": "Parents of kids aged 8–14 who already use one or more screen-time tools and find them inadequate.",
|
||||
"desiredOutcome": "A weekly conversation between parent and child about screen time becomes routine, grounded in shared data instead of conflict.",
|
||||
"initialHypothesis": "Parents will adopt a tool that's child-visible and child-collaborative if it lets them stop being the enforcer.",
|
||||
"constraints": [
|
||||
"Must work alongside iOS Screen Time and Google Family Link, not replace them",
|
||||
"Child data must be parent-controlled per COPPA (US) and GDPR-K (EU)",
|
||||
"No social features for kids — no leaderboards, no public sharing"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "medium",
|
||||
"testsFor": ["consumer-saas", "multi-actor", "regulatory"]
|
||||
}
|
||||
}
|
||||
19
phase-0/seeds/03-redline-ai.json
Normal file
19
phase-0/seeds/03-redline-ai.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"id": "03-redline-ai",
|
||||
"title": "Redline AI",
|
||||
"problem": "In-house legal teams spend hours per contract on first-pass redlining — tedious mechanical work that doesn't require senior judgment but does require consistent application of company-specific clause libraries. Junior associates burn out doing it; outside counsel charges $400/hr for it.",
|
||||
"targetUser": "General Counsel and Legal Operations leads at companies with 50–500 employees that handle 100+ contracts per quarter (SaaS, marketplaces, services).",
|
||||
"desiredOutcome": "First-pass redlines on standard contracts complete in minutes, with the GC reviewing AI suggestions instead of authoring redlines from scratch.",
|
||||
"initialHypothesis": "Legal teams will trust AI for first-pass mechanical redlining but reserve final judgment for humans — i.e., the bar is 'better than a tired junior associate', not 'better than the GC'.",
|
||||
"constraints": [
|
||||
"Must integrate with Microsoft Word — that's where the work happens",
|
||||
"Contract content must never train cross-customer models (privilege concerns)",
|
||||
"Audit trail of every change suggestion must be exportable",
|
||||
"SOC 2 Type II compliance from day one"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "medium",
|
||||
"testsFor": ["b2b", "domain-heavy", "regulatory", "integration"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/04-skillswap.json
Normal file
18
phase-0/seeds/04-skillswap.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "04-skillswap",
|
||||
"title": "SkillSwap",
|
||||
"problem": "Remote workers have skills they could exchange (a designer who needs a quick legal opinion; a lawyer who wants design help with a deck) but no good marketplace for non-cash skill trades. Cash marketplaces (Fiverr, Upwork) introduce a money layer that kills casual exchange.",
|
||||
"targetUser": "Remote knowledge workers, 25–45, who value reciprocal exchange over hiring out and have at least one marketable skill outside their day job.",
|
||||
"desiredOutcome": "Two strangers complete a skill swap end-to-end (matched, scoped, exchanged, both rated positively) within one week of joining.",
|
||||
"initialHypothesis": "A two-sided marketplace works for skill swap if matching is high-trust (reputation portable from LinkedIn) and scope is bounded (max 2-hour exchanges, no open-ended retainers).",
|
||||
"constraints": [
|
||||
"No money changes hands inside the platform",
|
||||
"Identity verification via LinkedIn or similar at signup",
|
||||
"Disputes capped to mediator review — no chargeback equivalent"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "medium",
|
||||
"testsFor": ["marketplace", "two-sided", "trust-mechanisms"]
|
||||
}
|
||||
}
|
||||
12
phase-0/seeds/05-cyclist-thing.json
Normal file
12
phase-0/seeds/05-cyclist-thing.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"id": "05-cyclist-thing",
|
||||
"title": "Cyclist Thing",
|
||||
"problem": "I want to build something for cyclists. Maybe an app, maybe a tool, not sure yet.",
|
||||
"targetUser": "Cyclists.",
|
||||
"desiredOutcome": "Something useful for them.",
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "failure-prone",
|
||||
"testsFor": ["vague-seed", "underspecified", "socrates-pushback"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/06-eventstream.json
Normal file
18
phase-0/seeds/06-eventstream.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "06-eventstream",
|
||||
"title": "EventStream",
|
||||
"problem": "Existing event-streaming systems (Kafka, Pulsar, Redpanda) treat backpressure as an afterthought — slow consumers either drop messages, lag indefinitely, or trigger broker-level fan-out failures. Modern application workloads (especially LLM-pipelined ones) need first-class backpressure semantics that the producer can observe in-band.",
|
||||
"targetUser": "Platform engineers at companies running event-driven infrastructure at moderate scale (100k–10M events/day) where consumer lag matters and Kafka's operational complexity is unwelcome.",
|
||||
"desiredOutcome": "A Go-native streaming system where producers receive backpressure signals as a first-class part of the protocol, and consumer-driven flow control is the default.",
|
||||
"initialHypothesis": "Teams will adopt a Kafka alternative if (a) operations are 10× simpler — single binary, no ZooKeeper, no Schema Registry — and (b) backpressure is a first-class protocol feature, not a config flag.",
|
||||
"constraints": [
|
||||
"Wire-compatible with Kafka producer protocol for migration safety",
|
||||
"Single binary, single config file, runs on a single node up to 100k events/sec",
|
||||
"Apache 2.0 license"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "hard",
|
||||
"testsFor": ["technical", "infra", "metamodel-strain"]
|
||||
}
|
||||
}
|
||||
17
phase-0/seeds/07-pet-translator.json
Normal file
17
phase-0/seeds/07-pet-translator.json
Normal file
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"id": "07-pet-translator",
|
||||
"title": "Pet Translator",
|
||||
"problem": "Pet owners want to know what their dogs and cats are 'saying'. Existing apps are gimmicks that produce random translations.",
|
||||
"targetUser": "Pet owners with dogs or cats, particularly first-time owners struggling to decode their pet's behavior.",
|
||||
"desiredOutcome": "Owners get a believable, useful interpretation of their pet's vocalization and body language — and use that interpretation to bond with their pet.",
|
||||
"initialHypothesis": "Recent advances in audio ML and computer vision make it possible to infer pet emotional state from vocalization + body language with sufficient accuracy that users perceive the output as 'translation' even if scientifically it's classification.",
|
||||
"constraints": [
|
||||
"On-device inference for privacy (no cloud upload of household audio/video)",
|
||||
"Honest about the difference between 'emotion classification' and 'translation' in onboarding"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "failure-prone",
|
||||
"testsFor": ["feasibility-suspect", "wishful-tech", "honesty-bar"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/08-quiet-hours.json
Normal file
18
phase-0/seeds/08-quiet-hours.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "08-quiet-hours",
|
||||
"title": "Quiet Hours",
|
||||
"problem": "Knowledge workers lose 2–3 hours per day to fragmented calendars — meetings scheduled mid-deep-work, no time long enough to make progress on anything substantial. Existing focus-time tools are either too rigid (block 9am–noon every day) or too easily overridden by colleagues.",
|
||||
"targetUser": "Individual contributors and engineering managers at 50+ person companies that use Google Calendar or Outlook and have collaborative cultures (lots of ad-hoc meetings).",
|
||||
"desiredOutcome": "Each user reliably gets at least 2 contiguous deep-work blocks per week without it costing them political capital.",
|
||||
"initialHypothesis": "If the tool can negotiate the protected blocks invisibly (proposing alternative meeting times that work for both parties), users get focus without the social cost of saying no.",
|
||||
"constraints": [
|
||||
"Read-write Google Calendar and Outlook (Microsoft Graph) integration",
|
||||
"Must respect timezone differences explicitly in proposals",
|
||||
"No automatic blocking without user confirmation for the first 30 days (build trust)"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": false,
|
||||
"expectedDifficulty": "medium",
|
||||
"testsFor": ["productivity", "calendar-integration", "constraint-heavy"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/09-carbon-coach.json
Normal file
18
phase-0/seeds/09-carbon-coach.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "09-carbon-coach",
|
||||
"title": "Carbon Coach",
|
||||
"problem": "Climate-aware consumers want to lower their personal carbon footprint but can't tell which lifestyle changes meaningfully matter and which are theatrical (e.g., reusable straws vs. flying less). Existing carbon calculators give one-shot estimates without sustained behavior change support.",
|
||||
"targetUser": "Climate-aware consumers in high-income countries, 25–55, already willing to change habits but unsure where to invest effort.",
|
||||
"desiredOutcome": "Sustained, measurable per-user reduction in monthly footprint over a 6-month period, driven by 2–3 high-impact behavior changes per user.",
|
||||
"initialHypothesis": "Social comparison via leaderboards (anonymous opt-in) drives behavior change more than absolute numbers — people respond to relative status more than abstract climate goals.",
|
||||
"constraints": [
|
||||
"Footprint estimates must cite their methodology (no black-box numbers)",
|
||||
"Anonymous social features only — no real-name sharing",
|
||||
"EU compliance with the upcoming Green Claims Directive"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": true,
|
||||
"expectedDifficulty": "medium",
|
||||
"testsFor": ["behavioral", "social-mechanics", "regulatory", "sustained-engagement"]
|
||||
}
|
||||
}
|
||||
18
phase-0/seeds/10-telemetry-lite.json
Normal file
18
phase-0/seeds/10-telemetry-lite.json
Normal file
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "10-telemetry-lite",
|
||||
"title": "Telemetry Lite",
|
||||
"problem": "PostHog has won the product-analytics-with-session-replay category but its self-hosted footprint is too heavy for small teams (Postgres, Kafka, ClickHouse, Redis, multiple workers). Privacy-conscious teams who want to self-host are stuck either running PostHog at $200/month of infra OR using nothing and getting blind product decisions.",
|
||||
"targetUser": "Engineering teams at 5–30 person companies (or solo founders) who want product analytics + session replay but refuse to send user data to a third-party SaaS, and don't have an infra engineer to run PostHog.",
|
||||
"desiredOutcome": "A single Docker container, single SQLite file, that delivers 80% of PostHog's value — events, funnels, session replay — at 5% of the operational complexity.",
|
||||
"initialHypothesis": "There's a self-host segment that won't pay for SaaS analytics OR run a PostHog cluster. They'll accept a smaller feature set in exchange for trivial ops.",
|
||||
"constraints": [
|
||||
"Single binary or single Docker container, SQLite-only storage option",
|
||||
"Apache 2.0 license, hosted version is a separate commercial play",
|
||||
"GDPR-compliant data export and deletion built in"
|
||||
],
|
||||
"meta": {
|
||||
"isHoldout": true,
|
||||
"expectedDifficulty": "hard",
|
||||
"testsFor": ["technical", "oss", "competitive-positioning"]
|
||||
}
|
||||
}
|
||||
341
phase-0/src/cli.ts
Normal file
341
phase-0/src/cli.ts
Normal file
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env node
|
||||
// Phase 0 CLI entrypoint.
|
||||
|
||||
import { Command } from 'commander';
|
||||
import kleur from 'kleur';
|
||||
import { describeClient, modelName } from './llm/client.js';
|
||||
import { chat } from './llm/messages.js';
|
||||
import { toPlantUML } from './render/to-plantuml.js';
|
||||
import { renderPNG, writePlantUMLSource, plantUMLViewerURL } from './render/render.js';
|
||||
import { aristotleReferenceModel } from './fixtures/aristotle.js';
|
||||
import { loadSeed } from './seed/load.js';
|
||||
import { generateModel } from './generate/generate.js';
|
||||
import { detectFindings } from './socrates/detect.js';
|
||||
import { runConversation } from './socrates/converse.js';
|
||||
import { proposeChange, applyProposal, summarizeOp } from './socrates/propose.js';
|
||||
import { runCorpus } from './eval/corpus.js';
|
||||
import { judgeSeed, averageScore, minDimension } from './eval/judge.js';
|
||||
import * as readline from 'node:readline/promises';
|
||||
import { stdin as stdInput, stdout as stdOutput } from 'node:process';
|
||||
import { resolve } from 'node:path';
|
||||
import { writeFile, mkdir } from 'node:fs/promises';
|
||||
|
||||
const program = new Command();
|
||||
program
|
||||
.name('phase0')
|
||||
.description('Socrata Phase 0 validation harness')
|
||||
.version('0.0.1');
|
||||
|
||||
program
|
||||
.command('hello')
|
||||
.description('Smoke test: round-trip a chat to LM Studio')
|
||||
.action(async () => {
|
||||
console.log(kleur.dim(`> ${describeClient()}`));
|
||||
const start = Date.now();
|
||||
try {
|
||||
const result = await chat([
|
||||
{ role: 'system', content: 'You are a terse assistant. Reply in one sentence.' },
|
||||
{ role: 'user', content: 'Are you there? Identify yourself briefly.' },
|
||||
], { temperature: 0.2, maxTokens: 256 });
|
||||
const elapsed = Date.now() - start;
|
||||
console.log(kleur.green('✓ round-trip OK'));
|
||||
console.log(kleur.dim(` ${elapsed}ms · ${result.inputTokens} in / ${result.outputTokens} out`));
|
||||
console.log();
|
||||
console.log(kleur.bold('response:'));
|
||||
console.log(result.text);
|
||||
} catch (err) {
|
||||
console.error(kleur.red('✗ round-trip FAILED'));
|
||||
console.error(kleur.red(` ${(err as Error).message}`));
|
||||
console.error(kleur.dim('\nIs LM Studio running? Is a model loaded?'));
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
|
||||
program
|
||||
.command('render-test')
|
||||
.description('Render the hand-written Aristotle reference model to PlantUML PNG')
|
||||
.option('-o, --out <dir>', 'output directory', './outputs/_render-test')
|
||||
.action(async (opts: { out: string }) => {
|
||||
const outDir = resolve(opts.out);
|
||||
const pumlPath = `${outDir}/aristotle.puml`;
|
||||
const pngPath = `${outDir}/aristotle.png`;
|
||||
|
||||
const source = toPlantUML(aristotleReferenceModel);
|
||||
await writePlantUMLSource(source, pumlPath);
|
||||
console.log(kleur.green('✓ wrote PlantUML source'));
|
||||
console.log(kleur.dim(` ${pumlPath}`));
|
||||
|
||||
try {
|
||||
await renderPNG(source, pngPath);
|
||||
console.log(kleur.green('✓ rendered PNG'));
|
||||
console.log(kleur.dim(` ${pngPath}`));
|
||||
console.log();
|
||||
console.log(kleur.bold('view online:'));
|
||||
console.log(plantUMLViewerURL(source));
|
||||
} catch (err) {
|
||||
console.error(kleur.yellow('⚠ PNG render failed (PlantUML source still saved):'));
|
||||
console.error(kleur.dim(` ${(err as Error).message}`));
|
||||
console.log();
|
||||
console.log(kleur.bold('view online:'));
|
||||
console.log(plantUMLViewerURL(source));
|
||||
}
|
||||
});
|
||||
|
||||
program
|
||||
.command('run <seedId>')
|
||||
.description('Run a seed through generation + render')
|
||||
.option('-o, --out <dir>', 'output directory (default: outputs/<seedId>)')
|
||||
.option('--converse', 'after detection, open an interactive Socrates conversation')
|
||||
.option('--propose', 'after detection, ask Socrates to propose a model change (accept/reject)')
|
||||
.action(async (seedId: string, opts: { out?: string; converse?: boolean; propose?: boolean }) => {
|
||||
const outDir = resolve(opts.out ?? `./outputs/${seedId}`);
|
||||
console.log(kleur.dim(`> seed: ${seedId}`));
|
||||
console.log(kleur.dim(`> ${describeClient()}`));
|
||||
console.log();
|
||||
|
||||
let seed;
|
||||
try {
|
||||
seed = await loadSeed(seedId);
|
||||
} catch (err) {
|
||||
console.error(kleur.red(`✗ couldn't load seed: ${(err as Error).message}`));
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(kleur.bold(`seed: ${seed.title}`));
|
||||
console.log(kleur.dim(` problem: ${truncate(seed.problem, 120)}`));
|
||||
console.log();
|
||||
|
||||
// Generate
|
||||
console.log(kleur.cyan('generating model…'));
|
||||
const gen = await generateModel(seed);
|
||||
console.log(kleur.green('✓ model generated'));
|
||||
console.log(
|
||||
kleur.dim(
|
||||
` ${gen.durationMs}ms · ${gen.inputTokens} in / ${gen.outputTokens} out · ` +
|
||||
`${gen.model.blocks.length} blocks · ${gen.model.associations.length} associations · ` +
|
||||
`${gen.model.constraints.length} constraints · ${gen.model.requirements.length} requirements · ` +
|
||||
`confidence ${gen.model.overallConfidence.toFixed(2)}`
|
||||
)
|
||||
);
|
||||
if (gen.warnings.length > 0) {
|
||||
console.log(kleur.yellow(' lint warnings:'));
|
||||
for (const w of gen.warnings) console.log(kleur.yellow(` · ${w}`));
|
||||
}
|
||||
console.log();
|
||||
|
||||
// Persist model + render in parallel with detection
|
||||
await mkdir(outDir, { recursive: true });
|
||||
await writeFile(`${outDir}/model.json`, JSON.stringify(gen.model, null, 2));
|
||||
|
||||
const source = toPlantUML(gen.model);
|
||||
await writePlantUMLSource(source, `${outDir}/diagram.puml`);
|
||||
|
||||
// Detection (3 prompts in parallel)
|
||||
console.log(kleur.cyan('detecting findings…'));
|
||||
const det = await detectFindings(gen.model, seed);
|
||||
const counts = countFindings(det.findings);
|
||||
console.log(kleur.green('✓ findings detected'));
|
||||
console.log(
|
||||
kleur.dim(
|
||||
` ${det.durationMs}ms · ${det.inputTokens} in / ${det.outputTokens} out · ` +
|
||||
`${counts.assumption} assumptions · ${counts.risk} risks · ${counts.inconsistency} inconsistencies`
|
||||
)
|
||||
);
|
||||
if (det.strippedRefs > 0 || det.droppedFindings > 0) {
|
||||
console.log(
|
||||
kleur.yellow(
|
||||
` · post-validate: ${det.strippedRefs} hallucinated refs stripped, ${det.droppedFindings} findings dropped`
|
||||
)
|
||||
);
|
||||
}
|
||||
await writeFile(`${outDir}/findings.json`, JSON.stringify(det.findings, null, 2));
|
||||
if (det.findings.length > 0) {
|
||||
console.log();
|
||||
printTopFindings(det.findings);
|
||||
}
|
||||
console.log();
|
||||
|
||||
try {
|
||||
await renderPNG(source, `${outDir}/diagram.png`);
|
||||
console.log(kleur.green('✓ rendered'));
|
||||
} catch (err) {
|
||||
console.log(kleur.yellow(`⚠ PNG render failed: ${(err as Error).message}`));
|
||||
}
|
||||
console.log(kleur.dim(` ${outDir}/`));
|
||||
console.log();
|
||||
console.log(kleur.bold('view diagram online:'));
|
||||
console.log(plantUMLViewerURL(source));
|
||||
|
||||
// Optional: Socrates proposes a model change (accept/reject)
|
||||
if (opts.propose) {
|
||||
console.log();
|
||||
console.log(kleur.dim('─'.repeat(60)));
|
||||
console.log(kleur.bold('Proposal'));
|
||||
console.log(kleur.dim('─'.repeat(60)));
|
||||
const proposal = await proposeChange(seed, gen.model, det.findings);
|
||||
console.log(kleur.bold('Σ Socrates proposes:'));
|
||||
console.log(` ${proposal.reasoning}`);
|
||||
console.log();
|
||||
if (proposal.ops.length === 0) {
|
||||
console.log(kleur.dim(' (no ops — nothing to apply)'));
|
||||
} else {
|
||||
console.log(kleur.bold('Ops:'));
|
||||
for (const op of proposal.ops) console.log(` ${summarizeOp(op)}`);
|
||||
console.log();
|
||||
const rl = readline.createInterface({ input: stdInput, output: stdOutput });
|
||||
let answer = '';
|
||||
try {
|
||||
answer = ((await rl.question(kleur.cyan('apply? [y/N] '))) ?? '').trim().toLowerCase();
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
if (answer === 'y' || answer === 'yes') {
|
||||
const applied = applyProposal(gen.model, proposal.ops);
|
||||
console.log(kleur.green(`✓ applied ${applied.applied.length} op(s)`));
|
||||
if (applied.skipped.length > 0) {
|
||||
console.log(kleur.yellow(` ⚠ skipped ${applied.skipped.length}:`));
|
||||
for (const s of applied.skipped) console.log(kleur.yellow(` - ${summarizeOp(s.op)} → ${s.reason}`));
|
||||
}
|
||||
await writeFile(`${outDir}/model.json`, JSON.stringify(applied.model, null, 2));
|
||||
const newSource = toPlantUML(applied.model);
|
||||
await writePlantUMLSource(newSource, `${outDir}/diagram.puml`);
|
||||
try {
|
||||
await renderPNG(newSource, `${outDir}/diagram.png`);
|
||||
console.log(kleur.green('✓ re-rendered'));
|
||||
console.log(kleur.dim(` ${outDir}/diagram.png`));
|
||||
console.log(kleur.bold('view diagram online:'));
|
||||
console.log(plantUMLViewerURL(newSource));
|
||||
} catch (err) {
|
||||
console.log(kleur.yellow(`⚠ PNG render failed: ${(err as Error).message}`));
|
||||
}
|
||||
} else {
|
||||
console.log(kleur.dim(' (proposal rejected)'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Optional: open an interactive Socrates conversation
|
||||
if (opts.converse) {
|
||||
console.log();
|
||||
console.log(kleur.dim('─'.repeat(60)));
|
||||
console.log(kleur.bold('Conversation with Socrates'));
|
||||
console.log(kleur.dim('Type :done or Ctrl+D to end · max 5 turns'));
|
||||
console.log(kleur.dim('─'.repeat(60)));
|
||||
|
||||
const conv = await runConversation(seed, gen.model, det.findings, {
|
||||
outPath: `${outDir}/conversation.md`,
|
||||
});
|
||||
console.log();
|
||||
console.log(kleur.green('✓ conversation saved'));
|
||||
console.log(
|
||||
kleur.dim(
|
||||
` ${conv.turns.length} turns · ${conv.durationMs}ms · ${conv.inputTokens} in / ${conv.outputTokens} out`
|
||||
)
|
||||
);
|
||||
console.log(kleur.dim(` ${outDir}/conversation.md`));
|
||||
}
|
||||
});
|
||||
|
||||
function countFindings(findings: { kind: string }[]): { assumption: number; risk: number; inconsistency: number } {
|
||||
const c = { assumption: 0, risk: 0, inconsistency: 0 };
|
||||
for (const f of findings) {
|
||||
if (f.kind in c) (c as Record<string, number>)[f.kind]++;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
function printTopFindings(findings: import('./types.js').Finding[]): void {
|
||||
const byKind = (k: string) => findings.filter(f => f.kind === k).slice(0, 3);
|
||||
const sections: Array<[string, string, ReturnType<typeof byKind>]> = [
|
||||
['assumptions', kleur.cyan('●'), byKind('assumption')],
|
||||
['risks', kleur.red('▲'), byKind('risk')],
|
||||
['inconsistencies', kleur.yellow('!'), byKind('inconsistency')],
|
||||
];
|
||||
for (const [label, glyph, items] of sections) {
|
||||
if (items.length === 0) continue;
|
||||
console.log(kleur.bold(` ${label}:`));
|
||||
for (const f of items) {
|
||||
const links = f.linkedElementIds.length > 0 ? kleur.dim(` [${f.linkedElementIds.join(', ')}]`) : '';
|
||||
const conf = kleur.dim(` (${f.confidence.toFixed(2)})`);
|
||||
console.log(` ${glyph} ${f.text}${links}${conf}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function truncate(s: string, n: number): string {
|
||||
return s.length > n ? s.slice(0, n - 1) + '…' : s;
|
||||
}
|
||||
|
||||
program
|
||||
.command('corpus')
|
||||
.description('Run all non-holdout seeds through generation + detection; write a report')
|
||||
.option('--final', 'include holdout seeds (final-round only; requires PHASE0_FINAL_ROUND=1)')
|
||||
.option('--skip-png', 'skip PNG rendering (faster — just .puml + JSON)')
|
||||
.option('--score', 'after corpus runs, score every seed with the LLM judge')
|
||||
.option('-o, --out <dir>', 'output root (default: ./outputs)')
|
||||
.option('-r, --report <file>', 'report path (default: ./reports/<timestamp>-run.md)')
|
||||
.action(async (opts: { final?: boolean; skipPng?: boolean; score?: boolean; out?: string; report?: string }) => {
|
||||
await runCorpus({
|
||||
includeHoldouts: !!opts.final,
|
||||
outRoot: opts.out ? resolve(opts.out) : undefined,
|
||||
reportPath: opts.report ? resolve(opts.report) : undefined,
|
||||
skipPNG: !!opts.skipPng,
|
||||
score: !!opts.score,
|
||||
});
|
||||
});
|
||||
|
||||
program
|
||||
.command('score <seedId>')
|
||||
.description('Score one seed\'s outputs with the LLM judge (writes outputs/<seedId>/score.json)')
|
||||
.action(async (seedId: string) => {
|
||||
console.log(kleur.dim(`> scoring ${seedId} with ${describeClient()}`));
|
||||
const judgeResult = await judgeSeed(seedId);
|
||||
const s = judgeResult.score;
|
||||
const avg = averageScore(s);
|
||||
const min = minDimension(s);
|
||||
console.log(kleur.green('✓ scored'));
|
||||
console.log(
|
||||
kleur.dim(
|
||||
` ${judgeResult.durationMs}ms · ${judgeResult.inputTokens} in / ${judgeResult.outputTokens} out · ` +
|
||||
`avg ${avg.toFixed(2)} · min ${min}`
|
||||
)
|
||||
);
|
||||
console.log();
|
||||
console.log(kleur.bold('Scores:'));
|
||||
const dims: Array<[keyof typeof judgeResult.reasons, string]> = [
|
||||
['modelCoverage', 'model.coverage '],
|
||||
['modelAccuracy', 'model.accuracy '],
|
||||
['modelParsimony', 'model.parsimony '],
|
||||
['constraintCapture', 'constraint.capture'],
|
||||
['assumptionDetectionQuality', 'asm.detection '],
|
||||
['riskDetectionQuality', 'risk.detection '],
|
||||
['voiceAndCharacter', 'voice.character '],
|
||||
];
|
||||
for (const [key, label] of dims) {
|
||||
const v = (s as unknown as Record<string, number>)[key]!;
|
||||
const bar = '█'.repeat(v) + '░'.repeat(5 - v);
|
||||
const color = v >= 4 ? kleur.green : v >= 3 ? kleur.yellow : kleur.red;
|
||||
console.log(` ${label} ${color(bar)} ${v} ${kleur.dim('— ' + judgeResult.reasons[key])}`);
|
||||
}
|
||||
if (judgeResult.overallComment) {
|
||||
console.log();
|
||||
console.log(kleur.bold('Comment:'), judgeResult.overallComment);
|
||||
}
|
||||
});
|
||||
|
||||
program
|
||||
.command('config')
|
||||
.description('Print resolved configuration')
|
||||
.action(() => {
|
||||
console.log(kleur.bold('LM Studio'));
|
||||
console.log(` baseURL: ${process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1'}`);
|
||||
console.log(` model: ${modelName}`);
|
||||
console.log(` apiKey: ${process.env.LMSTUDIO_API_KEY ? kleur.green('set') : kleur.yellow('default (lm-studio)')}`);
|
||||
console.log();
|
||||
console.log(kleur.bold('PlantUML'));
|
||||
console.log(` server: ${process.env.PLANTUML_SERVER ?? 'https://www.plantuml.com/plantuml'}`);
|
||||
});
|
||||
|
||||
program.parseAsync(process.argv).catch(err => {
|
||||
console.error(kleur.red(err instanceof Error ? err.message : String(err)));
|
||||
process.exit(1);
|
||||
});
|
||||
323
phase-0/src/eval/corpus.ts
Normal file
323
phase-0/src/eval/corpus.ts
Normal file
@@ -0,0 +1,323 @@
|
||||
// Run the corpus through generation + detection and write a summary report.
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import kleur from 'kleur';
|
||||
import { loadCorpus } from '../seed/load.js';
|
||||
import { generateModel } from '../generate/generate.js';
|
||||
import { detectFindings } from '../socrates/detect.js';
|
||||
import { toPlantUML } from '../render/to-plantuml.js';
|
||||
import { renderPNG, writePlantUMLSource, plantUMLViewerURL } from '../render/render.js';
|
||||
import { lintModel } from '../generate/schema.js';
|
||||
import { modelName } from '../llm/client.js';
|
||||
import { judgeArtifacts, averageScore, minDimension, type JudgeResult } from './judge.js';
|
||||
import type { GeneratedModel, Finding, RubricScore, SeedPayload } from '../types.js';
|
||||
|
||||
export interface CorpusRunOptions {
|
||||
includeHoldouts: boolean;
|
||||
outRoot?: string;
|
||||
reportPath?: string;
|
||||
/** Skip PNG rendering (faster, just .puml + JSON). */
|
||||
skipPNG?: boolean;
|
||||
/** After detection, score every seed with the LLM judge. */
|
||||
score?: boolean;
|
||||
}
|
||||
|
||||
interface PerSeedResult {
|
||||
seedId: string;
|
||||
title: string;
|
||||
difficulty: SeedPayload['meta']['expectedDifficulty'];
|
||||
isHoldout: boolean;
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
model?: GeneratedModel;
|
||||
findings?: Finding[];
|
||||
durations?: { generateMs: number; detectMs: number; scoreMs?: number };
|
||||
tokens?: { genIn: number; genOut: number; detIn: number; detOut: number; scoreIn?: number; scoreOut?: number };
|
||||
lintWarnings?: string[];
|
||||
postValidate?: { strippedRefs: number; droppedFindings: number };
|
||||
plantUmlUrl?: string;
|
||||
judge?: JudgeResult;
|
||||
}
|
||||
|
||||
export async function runCorpus(opts: CorpusRunOptions): Promise<PerSeedResult[]> {
|
||||
const seeds = await loadCorpus({ includeHoldouts: opts.includeHoldouts });
|
||||
const outRoot = opts.outRoot ?? resolve(process.cwd(), 'outputs');
|
||||
const results: PerSeedResult[] = [];
|
||||
|
||||
console.log(kleur.bold(`\nRunning corpus (${seeds.length} seeds)`));
|
||||
console.log(kleur.dim(`Model: ${modelName}\n`));
|
||||
|
||||
for (let i = 0; i < seeds.length; i++) {
|
||||
const seed = seeds[i]!;
|
||||
const tag = `[${i + 1}/${seeds.length}] ${seed.id}`;
|
||||
process.stdout.write(`${kleur.bold(tag)} ${kleur.dim('· ' + seed.title)}\n`);
|
||||
|
||||
const outDir = `${outRoot}/${seed.id}`;
|
||||
await mkdir(outDir, { recursive: true });
|
||||
|
||||
try {
|
||||
const gen = await generateModel(seed);
|
||||
const lint = lintModel(gen.model);
|
||||
await writeFile(`${outDir}/model.json`, JSON.stringify(gen.model, null, 2));
|
||||
|
||||
const source = toPlantUML(gen.model);
|
||||
await writePlantUMLSource(source, `${outDir}/diagram.puml`);
|
||||
const url = plantUMLViewerURL(source);
|
||||
|
||||
if (!opts.skipPNG) {
|
||||
try {
|
||||
await renderPNG(source, `${outDir}/diagram.png`);
|
||||
} catch (err) {
|
||||
console.log(kleur.yellow(` ⚠ PNG render failed: ${(err as Error).message}`));
|
||||
}
|
||||
}
|
||||
|
||||
const det = await detectFindings(gen.model, seed);
|
||||
await writeFile(`${outDir}/findings.json`, JSON.stringify(det.findings, null, 2));
|
||||
|
||||
const counts = countFindings(det.findings);
|
||||
|
||||
// Optional: LLM judge
|
||||
let judgeResult: JudgeResult | undefined;
|
||||
if (opts.score) {
|
||||
try {
|
||||
judgeResult = await judgeArtifacts(seed, gen.model, det.findings, outDir);
|
||||
} catch (err) {
|
||||
console.log(kleur.yellow(` ⚠ judge failed: ${(err as Error).message.split('\n')[0]}`));
|
||||
}
|
||||
}
|
||||
|
||||
const scoreSuffix = judgeResult
|
||||
? ` · score ${averageScore(judgeResult.score).toFixed(2)} (min ${minDimension(judgeResult.score)})`
|
||||
: '';
|
||||
console.log(
|
||||
kleur.dim(
|
||||
` ${gen.model.blocks.length}b ${gen.model.associations.length}a ${gen.model.constraints.length}c ${gen.model.requirements.length}r · ` +
|
||||
`${counts.assumption}A ${counts.risk}R ${counts.inconsistency}I · ` +
|
||||
`lint=${lint.warnings.length} · stripped=${det.strippedRefs} · ` +
|
||||
`${(gen.durationMs / 1000).toFixed(1)}s gen + ${(det.durationMs / 1000).toFixed(1)}s det` +
|
||||
(judgeResult ? ` + ${(judgeResult.durationMs / 1000).toFixed(1)}s judge` : '') +
|
||||
` · conf ${gen.model.overallConfidence.toFixed(2)}` +
|
||||
scoreSuffix
|
||||
)
|
||||
);
|
||||
|
||||
results.push({
|
||||
seedId: seed.id,
|
||||
title: seed.title,
|
||||
difficulty: seed.meta.expectedDifficulty,
|
||||
isHoldout: seed.meta.isHoldout,
|
||||
ok: true,
|
||||
model: gen.model,
|
||||
findings: det.findings,
|
||||
durations: {
|
||||
generateMs: gen.durationMs,
|
||||
detectMs: det.durationMs,
|
||||
...(judgeResult ? { scoreMs: judgeResult.durationMs } : {}),
|
||||
},
|
||||
tokens: {
|
||||
genIn: gen.inputTokens,
|
||||
genOut: gen.outputTokens,
|
||||
detIn: det.inputTokens,
|
||||
detOut: det.outputTokens,
|
||||
...(judgeResult ? { scoreIn: judgeResult.inputTokens, scoreOut: judgeResult.outputTokens } : {}),
|
||||
},
|
||||
lintWarnings: lint.warnings,
|
||||
postValidate: { strippedRefs: det.strippedRefs, droppedFindings: det.droppedFindings },
|
||||
plantUmlUrl: url,
|
||||
judge: judgeResult,
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = (err as Error).message ?? String(err);
|
||||
console.log(kleur.red(` ✗ failed: ${msg.split('\n')[0]}`));
|
||||
results.push({
|
||||
seedId: seed.id,
|
||||
title: seed.title,
|
||||
difficulty: seed.meta.expectedDifficulty,
|
||||
isHoldout: seed.meta.isHoldout,
|
||||
ok: false,
|
||||
error: msg,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Write the report.
|
||||
const reportPath = opts.reportPath ?? resolve(process.cwd(), `reports/${dateStamp()}-run.md`);
|
||||
await mkdir(resolve(reportPath, '..'), { recursive: true });
|
||||
await writeFile(reportPath, renderReport(results), 'utf8');
|
||||
console.log();
|
||||
console.log(kleur.green('✓ corpus run complete'));
|
||||
console.log(kleur.dim(` report: ${reportPath}`));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function countFindings(findings: Finding[]): { assumption: number; risk: number; inconsistency: number } {
|
||||
return {
|
||||
assumption: findings.filter(f => f.kind === 'assumption').length,
|
||||
risk: findings.filter(f => f.kind === 'risk').length,
|
||||
inconsistency: findings.filter(f => f.kind === 'inconsistency').length,
|
||||
};
|
||||
}
|
||||
|
||||
function dateStamp(): string {
|
||||
const d = new Date();
|
||||
return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}`;
|
||||
}
|
||||
function pad(n: number): string {
|
||||
return n.toString().padStart(2, '0');
|
||||
}
|
||||
|
||||
function renderReport(results: PerSeedResult[]): string {
|
||||
const lines: string[] = [];
|
||||
lines.push(`# Phase 0 corpus run · ${new Date().toISOString()}`);
|
||||
lines.push('');
|
||||
lines.push(`**Model:** \`${modelName}\``);
|
||||
const ok = results.filter(r => r.ok).length;
|
||||
lines.push(`**Ran:** ${results.length} · **OK:** ${ok} · **Failed:** ${results.length - ok}`);
|
||||
lines.push('');
|
||||
lines.push('## Summary');
|
||||
lines.push('');
|
||||
const anyScored = results.some(r => r.judge);
|
||||
if (anyScored) {
|
||||
lines.push('| Seed | Diff | Blocks/A/C/R | A/R/I | Conf | Cov | Acc | Par | Cstr | Asm | Risk | Voice | **Avg** | **Min** |');
|
||||
lines.push('|---|---|---|---|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const r of results) {
|
||||
if (!r.ok) {
|
||||
lines.push(`| ${r.seedId} | ${r.difficulty} | ❌ | — | — | — | — | — | — | — | — | — | — | — |`);
|
||||
continue;
|
||||
}
|
||||
const m = r.model!;
|
||||
const c = countFindings(r.findings!);
|
||||
const counts = `${m.blocks.length}/${m.associations.length}/${m.constraints.length}/${m.requirements.length}`;
|
||||
const detected = `${c.assumption}/${c.risk}/${c.inconsistency}`;
|
||||
const j = r.judge;
|
||||
if (!j) {
|
||||
lines.push(`| ${r.seedId} | ${r.difficulty} | ${counts} | ${detected} | ${m.overallConfidence.toFixed(2)} | — | — | — | — | — | — | — | — | — |`);
|
||||
} else {
|
||||
const s = j.score;
|
||||
const avg = averageScore(s).toFixed(2);
|
||||
const min = minDimension(s);
|
||||
lines.push(
|
||||
`| ${r.seedId} | ${r.difficulty} | ${counts} | ${detected} | ${m.overallConfidence.toFixed(2)} | ${s.modelCoverage} | ${s.modelAccuracy} | ${s.modelParsimony} | ${s.constraintCapture} | ${s.assumptionDetectionQuality} | ${s.riskDetectionQuality} | ${s.voiceAndCharacter} | **${avg}** | **${min}** |`
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
lines.push('| Seed | Diff | Blocks | Assoc | Constr | Req | Asm | Risk | Inco | Lint | Stripped | Conf | Gen (s) | Det (s) | Status |');
|
||||
lines.push('|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const r of results) {
|
||||
if (!r.ok) {
|
||||
lines.push(`| ${r.seedId} | ${r.difficulty} | — | — | — | — | — | — | — | — | — | — | — | — | ❌ |`);
|
||||
continue;
|
||||
}
|
||||
const m = r.model!;
|
||||
const c = countFindings(r.findings!);
|
||||
lines.push(
|
||||
`| ${r.seedId} | ${r.difficulty} | ${m.blocks.length} | ${m.associations.length} | ${m.constraints.length} | ${m.requirements.length} | ${c.assumption} | ${c.risk} | ${c.inconsistency} | ${r.lintWarnings!.length} | ${r.postValidate!.strippedRefs} | ${m.overallConfidence.toFixed(2)} | ${(r.durations!.generateMs / 1000).toFixed(1)} | ${(r.durations!.detectMs / 1000).toFixed(1)} | ✓ |`
|
||||
);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Pass/fail roll-up if scored
|
||||
if (anyScored) {
|
||||
const scored = results.filter(r => r.judge);
|
||||
const passing = scored.filter(r => {
|
||||
const s = r.judge!.score;
|
||||
return averageScore(s) >= 3.5 && minDimension(s) >= 3;
|
||||
});
|
||||
lines.push(`**Rubric pass criteria:** avg ≥ 3.5 AND min ≥ 3 per seed`);
|
||||
lines.push('');
|
||||
lines.push(`**Passing:** ${passing.length} / ${scored.length}`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Per-seed details
|
||||
lines.push('## Per-seed detail');
|
||||
lines.push('');
|
||||
for (const r of results) {
|
||||
lines.push(`### ${r.seedId} — ${r.title}`);
|
||||
lines.push('');
|
||||
if (!r.ok) {
|
||||
lines.push('**Status:** ❌ failed');
|
||||
lines.push('');
|
||||
lines.push('```');
|
||||
lines.push((r.error ?? '').slice(0, 1000));
|
||||
lines.push('```');
|
||||
lines.push('');
|
||||
continue;
|
||||
}
|
||||
const m = r.model!;
|
||||
const f = r.findings!;
|
||||
lines.push(`- **Difficulty:** ${r.difficulty}${r.isHoldout ? ' · HOLDOUT' : ''}`);
|
||||
lines.push(`- **Confidence:** ${m.overallConfidence.toFixed(2)}`);
|
||||
lines.push(`- **Diagram:** [view online](${r.plantUmlUrl})`);
|
||||
if (m.notes) lines.push(`- **Notes:** ${m.notes}`);
|
||||
if (r.lintWarnings!.length > 0) {
|
||||
lines.push(`- **Lint warnings:** ${r.lintWarnings!.length}`);
|
||||
for (const w of r.lintWarnings!) lines.push(` - ${w}`);
|
||||
}
|
||||
if (r.postValidate!.strippedRefs > 0 || r.postValidate!.droppedFindings > 0) {
|
||||
lines.push(`- **Post-validate:** stripped ${r.postValidate!.strippedRefs} refs, dropped ${r.postValidate!.droppedFindings} findings`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('**Model elements:**');
|
||||
lines.push('');
|
||||
lines.push(`- Blocks: ${m.blocks.map(b => `${b.label} (${b.kind})`).join(', ') || '_none_'}`);
|
||||
lines.push(`- Associations: ${m.associations.map(a => `${a.fromBlockId}→${a.toBlockId}:${a.label}`).join(', ') || '_none_'}`);
|
||||
lines.push(`- Constraints: ${m.constraints.map(c => c.label).join(', ') || '_none_'}`);
|
||||
lines.push(`- Requirements: ${m.requirements.map(r => r.tag).join(', ') || '_none_'}`);
|
||||
lines.push('');
|
||||
|
||||
const byKind = (k: Finding['kind']) => f.filter(x => x.kind === k);
|
||||
for (const [kind, label] of [
|
||||
['assumption', 'Assumptions'],
|
||||
['risk', 'Risks'],
|
||||
['inconsistency', 'Inconsistencies'],
|
||||
] as const) {
|
||||
const items = byKind(kind);
|
||||
if (items.length === 0) continue;
|
||||
lines.push(`**${label}:**`);
|
||||
lines.push('');
|
||||
for (const it of items) {
|
||||
const refs = it.linkedElementIds.length > 0 ? ` [${it.linkedElementIds.join(', ')}]` : '';
|
||||
const sev = it.severity ? ` · ${it.severity}` : '';
|
||||
lines.push(`- (${it.confidence.toFixed(2)}${sev}) ${it.text}${refs}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Judge scores
|
||||
if (r.judge) {
|
||||
const j = r.judge;
|
||||
const s = j.score;
|
||||
lines.push('**Judge scores** (LLM-as-judge — sanity check, not final):');
|
||||
lines.push('');
|
||||
const dims: Array<[keyof typeof j.reasons, string, number]> = [
|
||||
['modelCoverage', 'model.coverage', s.modelCoverage],
|
||||
['modelAccuracy', 'model.accuracy', s.modelAccuracy],
|
||||
['modelParsimony', 'model.parsimony', s.modelParsimony],
|
||||
['constraintCapture', 'constraint.capture', s.constraintCapture],
|
||||
['assumptionDetectionQuality', 'asm.detection', s.assumptionDetectionQuality],
|
||||
['riskDetectionQuality', 'risk.detection', s.riskDetectionQuality],
|
||||
['voiceAndCharacter', 'voice.character', s.voiceAndCharacter],
|
||||
];
|
||||
for (const [key, label, v] of dims) {
|
||||
lines.push(`- **${label}:** ${v}/5 — ${j.reasons[key]}`);
|
||||
}
|
||||
lines.push(`- **Average:** ${averageScore(s).toFixed(2)} · **Min:** ${minDimension(s)}`);
|
||||
if (j.overallComment) {
|
||||
lines.push('');
|
||||
lines.push(`> ${j.overallComment}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
270
phase-0/src/eval/judge.ts
Normal file
270
phase-0/src/eval/judge.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
// LLM-as-judge — scores a single seed's run on the 7-dimension rubric.
|
||||
|
||||
import { z } from 'zod';
|
||||
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { chatJSON, type Message } from '../llm/messages.js';
|
||||
import { loadPrompt } from '../prompts/load.js';
|
||||
import { loadSeed } from '../seed/load.js';
|
||||
import { modelName } from '../llm/client.js';
|
||||
import type { GeneratedModel, Finding, RubricScore, SeedPayload } from '../types.js';
|
||||
|
||||
// ─── Schemas ─────────────────────────────────────────────────────────────
|
||||
|
||||
const ScoreInt = z.number().int().min(1).max(5);
|
||||
|
||||
const JudgeResponse = z.object({
|
||||
modelCoverage: ScoreInt,
|
||||
modelCoverageReason: z.string().min(1),
|
||||
modelAccuracy: ScoreInt,
|
||||
modelAccuracyReason: z.string().min(1),
|
||||
modelParsimony: ScoreInt,
|
||||
modelParsimonyReason: z.string().min(1),
|
||||
constraintCapture: ScoreInt,
|
||||
constraintCaptureReason: z.string().min(1),
|
||||
assumptionDetectionQuality: ScoreInt,
|
||||
assumptionDetectionQualityReason: z.string().min(1),
|
||||
riskDetectionQuality: ScoreInt,
|
||||
riskDetectionQualityReason: z.string().min(1),
|
||||
voiceAndCharacter: ScoreInt,
|
||||
voiceAndCharacterReason: z.string().min(1),
|
||||
overallComment: z.string().optional(),
|
||||
});
|
||||
|
||||
type JudgeResponse = z.infer<typeof JudgeResponse>;
|
||||
|
||||
const judgeJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: [
|
||||
'modelCoverage',
|
||||
'modelCoverageReason',
|
||||
'modelAccuracy',
|
||||
'modelAccuracyReason',
|
||||
'modelParsimony',
|
||||
'modelParsimonyReason',
|
||||
'constraintCapture',
|
||||
'constraintCaptureReason',
|
||||
'assumptionDetectionQuality',
|
||||
'assumptionDetectionQualityReason',
|
||||
'riskDetectionQuality',
|
||||
'riskDetectionQualityReason',
|
||||
'voiceAndCharacter',
|
||||
'voiceAndCharacterReason',
|
||||
],
|
||||
properties: {
|
||||
modelCoverage: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
modelCoverageReason: { type: 'string', minLength: 1 },
|
||||
modelAccuracy: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
modelAccuracyReason: { type: 'string', minLength: 1 },
|
||||
modelParsimony: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
modelParsimonyReason: { type: 'string', minLength: 1 },
|
||||
constraintCapture: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
constraintCaptureReason: { type: 'string', minLength: 1 },
|
||||
assumptionDetectionQuality: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
assumptionDetectionQualityReason: { type: 'string', minLength: 1 },
|
||||
riskDetectionQuality: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
riskDetectionQualityReason: { type: 'string', minLength: 1 },
|
||||
voiceAndCharacter: { type: 'integer', minimum: 1, maximum: 5 },
|
||||
voiceAndCharacterReason: { type: 'string', minLength: 1 },
|
||||
overallComment: { type: 'string' },
|
||||
},
|
||||
} as const;
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface JudgeOptions {
|
||||
/** Optional output dir (defaults to outputs/<seedId>) */
|
||||
outDir?: string;
|
||||
}
|
||||
|
||||
export interface JudgeResult {
|
||||
score: RubricScore;
|
||||
reasons: {
|
||||
modelCoverage: string;
|
||||
modelAccuracy: string;
|
||||
modelParsimony: string;
|
||||
constraintCapture: string;
|
||||
assumptionDetectionQuality: string;
|
||||
riskDetectionQuality: string;
|
||||
voiceAndCharacter: string;
|
||||
};
|
||||
overallComment?: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export async function judgeSeed(
|
||||
seedId: string,
|
||||
opts: JudgeOptions = {}
|
||||
): Promise<JudgeResult> {
|
||||
const outDir = opts.outDir ?? resolve(process.cwd(), `outputs/${seedId}`);
|
||||
if (!existsSync(`${outDir}/model.json`)) {
|
||||
throw new Error(`No model.json found at ${outDir}/model.json — run \`phase0 run ${seedId}\` first`);
|
||||
}
|
||||
if (!existsSync(`${outDir}/findings.json`)) {
|
||||
throw new Error(`No findings.json found at ${outDir}/findings.json`);
|
||||
}
|
||||
|
||||
const seed = await loadSeed(seedId);
|
||||
const model: GeneratedModel = JSON.parse(await readFile(`${outDir}/model.json`, 'utf8'));
|
||||
const findings: Finding[] = JSON.parse(await readFile(`${outDir}/findings.json`, 'utf8'));
|
||||
|
||||
return judgeArtifacts(seed, model, findings, outDir);
|
||||
}
|
||||
|
||||
export async function judgeArtifacts(
|
||||
seed: SeedPayload,
|
||||
model: GeneratedModel,
|
||||
findings: Finding[],
|
||||
outDir: string
|
||||
): Promise<JudgeResult> {
|
||||
const judge = await loadPrompt('judge');
|
||||
|
||||
const userPayload =
|
||||
`Seed:\n\`\`\`json\n${JSON.stringify(stripMeta(seed), null, 2)}\n\`\`\`\n\n` +
|
||||
`Seed difficulty: **${seed.meta.expectedDifficulty}** (testsFor: ${seed.meta.testsFor.join(', ')})\n\n` +
|
||||
`Generated model:\n\`\`\`json\n${JSON.stringify(trimModel(model), null, 2)}\n\`\`\`\n\n` +
|
||||
`Detected findings:\n\`\`\`json\n${JSON.stringify(trimFindings(findings), null, 2)}\n\`\`\`\n`;
|
||||
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: judge },
|
||||
{ role: 'user', content: userPayload },
|
||||
];
|
||||
|
||||
const start = Date.now();
|
||||
const { value, result } = await chatJSON<unknown>(messages, {
|
||||
temperature: 0.2,
|
||||
maxTokens: 1500,
|
||||
jsonSchema: { name: 'rubric_score', schema: judgeJsonSchema as Record<string, unknown> },
|
||||
useSchema: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
|
||||
const parsed = JudgeResponse.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
throw new Error(
|
||||
`Judge JSON failed schema validation:\n` +
|
||||
parsed.error.issues.map(i => ` ${i.path.join('.')}: ${i.message}`).join('\n')
|
||||
);
|
||||
}
|
||||
|
||||
const score: RubricScore = {
|
||||
modelCoverage: parsed.data.modelCoverage,
|
||||
modelAccuracy: parsed.data.modelAccuracy,
|
||||
modelParsimony: parsed.data.modelParsimony,
|
||||
constraintCapture: parsed.data.constraintCapture,
|
||||
assumptionDetectionQuality: parsed.data.assumptionDetectionQuality,
|
||||
riskDetectionQuality: parsed.data.riskDetectionQuality,
|
||||
voiceAndCharacter: parsed.data.voiceAndCharacter,
|
||||
comments: parsed.data.overallComment,
|
||||
scorer: 'llm-judge',
|
||||
scorerName: modelName,
|
||||
};
|
||||
|
||||
const judgeResult: JudgeResult = {
|
||||
score,
|
||||
reasons: {
|
||||
modelCoverage: parsed.data.modelCoverageReason,
|
||||
modelAccuracy: parsed.data.modelAccuracyReason,
|
||||
modelParsimony: parsed.data.modelParsimonyReason,
|
||||
constraintCapture: parsed.data.constraintCaptureReason,
|
||||
assumptionDetectionQuality: parsed.data.assumptionDetectionQualityReason,
|
||||
riskDetectionQuality: parsed.data.riskDetectionQualityReason,
|
||||
voiceAndCharacter: parsed.data.voiceAndCharacterReason,
|
||||
},
|
||||
overallComment: parsed.data.overallComment,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
|
||||
// Persist
|
||||
await mkdir(dirname(`${outDir}/score.json`), { recursive: true });
|
||||
await writeFile(
|
||||
`${outDir}/score.json`,
|
||||
JSON.stringify(judgeResult, null, 2),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
return judgeResult;
|
||||
}
|
||||
|
||||
export function averageScore(s: RubricScore): number {
|
||||
return (
|
||||
(s.modelCoverage +
|
||||
s.modelAccuracy +
|
||||
s.modelParsimony +
|
||||
s.constraintCapture +
|
||||
s.assumptionDetectionQuality +
|
||||
s.riskDetectionQuality +
|
||||
s.voiceAndCharacter) /
|
||||
7
|
||||
);
|
||||
}
|
||||
|
||||
export function minDimension(s: RubricScore): number {
|
||||
return Math.min(
|
||||
s.modelCoverage,
|
||||
s.modelAccuracy,
|
||||
s.modelParsimony,
|
||||
s.constraintCapture,
|
||||
s.assumptionDetectionQuality,
|
||||
s.riskDetectionQuality,
|
||||
s.voiceAndCharacter
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
function trimModel(model: GeneratedModel): unknown {
|
||||
return {
|
||||
systemOfInterestId: model.systemOfInterestId,
|
||||
blocks: model.blocks.map(b => ({
|
||||
id: b.id,
|
||||
label: b.label,
|
||||
kind: b.kind,
|
||||
properties: b.properties.map(p => p.name),
|
||||
})),
|
||||
associations: model.associations.map(a => ({
|
||||
id: a.id,
|
||||
from: a.fromBlockId,
|
||||
to: a.toBlockId,
|
||||
label: a.label,
|
||||
kind: a.kind,
|
||||
})),
|
||||
constraints: model.constraints.map(c => ({
|
||||
id: c.id,
|
||||
label: c.label,
|
||||
appliesTo: c.appliesTo,
|
||||
})),
|
||||
requirements: model.requirements.map(r => ({
|
||||
id: r.id,
|
||||
tag: r.tag,
|
||||
text: r.text,
|
||||
satisfiedBy: r.relations
|
||||
.filter(rel => rel.kind === 'satisfy')
|
||||
.map(rel => (rel as { kind: 'satisfy'; blockId: string }).blockId),
|
||||
})),
|
||||
overallConfidence: model.overallConfidence,
|
||||
notes: model.notes,
|
||||
};
|
||||
}
|
||||
|
||||
function trimFindings(findings: Finding[]): unknown {
|
||||
return findings.map(f => ({
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
refs: f.linkedElementIds,
|
||||
...(f.severity ? { severity: f.severity } : {}),
|
||||
confidence: f.confidence,
|
||||
}));
|
||||
}
|
||||
|
||||
function stripMeta(seed: SeedPayload): Omit<SeedPayload, 'meta'> {
|
||||
const { meta, ...rest } = seed;
|
||||
void meta;
|
||||
return rest;
|
||||
}
|
||||
151
phase-0/src/fixtures/aristotle.ts
Normal file
151
phase-0/src/fixtures/aristotle.ts
Normal file
@@ -0,0 +1,151 @@
|
||||
// Hand-written reference model for the Aristotle seed.
|
||||
// Used as a fixture for `phase0 render-test` and as a known-good baseline
|
||||
// when evaluating LLM-generated models.
|
||||
|
||||
import type { GeneratedModel } from '../types.js';
|
||||
|
||||
export const aristotleReferenceModel: GeneratedModel = {
|
||||
systemOfInterestId: 'aristotle',
|
||||
overallConfidence: 1.0,
|
||||
notes: 'Hand-authored reference model for the Aristotle seed.',
|
||||
blocks: [
|
||||
{
|
||||
id: 'aristotle',
|
||||
label: 'Aristotle',
|
||||
kind: 'system',
|
||||
stereotypes: ['system'],
|
||||
confidence: 1.0,
|
||||
properties: [
|
||||
{ id: 'p1', name: 'refusal_policy', type: { kind: 'string' }, multiplicity: '1' },
|
||||
{ id: 'p2', name: 'interaction_style', type: { kind: 'enum', values: ['socratic', 'tutor'] }, multiplicity: '1' },
|
||||
{ id: 'p3', name: 'scope_window', type: { kind: 'string' }, multiplicity: '1' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'student',
|
||||
label: 'Student',
|
||||
kind: 'actor',
|
||||
stereotypes: ['actor'],
|
||||
confidence: 1.0,
|
||||
properties: [
|
||||
{ id: 'p4', name: 'self_efficacy', type: { kind: 'number' }, multiplicity: '0..1' },
|
||||
{ id: 'p5', name: 'course_load', type: { kind: 'number' }, multiplicity: '0..1' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'instructor',
|
||||
label: 'Instructor',
|
||||
kind: 'actor',
|
||||
stereotypes: ['actor'],
|
||||
confidence: 0.9,
|
||||
properties: [
|
||||
{ id: 'p6', name: 'policy_set', type: { kind: 'string' }, multiplicity: '0..*' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'course',
|
||||
label: 'Course',
|
||||
kind: 'block',
|
||||
stereotypes: ['block'],
|
||||
confidence: 1.0,
|
||||
properties: [
|
||||
{ id: 'p7', name: 'syllabus', type: { kind: 'string' }, multiplicity: '0..1' },
|
||||
{ id: 'p8', name: 'prerequisites', type: { kind: 'string' }, multiplicity: '0..*' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'assignment',
|
||||
label: 'Assignment',
|
||||
kind: 'block',
|
||||
stereotypes: ['block'],
|
||||
confidence: 1.0,
|
||||
properties: [
|
||||
{ id: 'p9', name: 'due_at', type: { kind: 'string' }, multiplicity: '1' },
|
||||
{ id: 'p10', name: 'rubric', type: { kind: 'string' }, multiplicity: '0..1' },
|
||||
{ id: 'p11', name: 'graded', type: { kind: 'boolean' }, multiplicity: '1' },
|
||||
],
|
||||
},
|
||||
],
|
||||
associations: [
|
||||
{
|
||||
id: 'a1',
|
||||
fromBlockId: 'student',
|
||||
toBlockId: 'aristotle',
|
||||
label: 'consults',
|
||||
kind: 'association',
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'a2',
|
||||
fromBlockId: 'aristotle',
|
||||
toBlockId: 'assignment',
|
||||
label: 'scoped_to',
|
||||
kind: 'association',
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'a3',
|
||||
fromBlockId: 'course',
|
||||
toBlockId: 'assignment',
|
||||
label: 'contains',
|
||||
kind: 'composition',
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'a4',
|
||||
fromBlockId: 'instructor',
|
||||
toBlockId: 'aristotle',
|
||||
label: 'configures',
|
||||
kind: 'association',
|
||||
confidence: 0.85,
|
||||
},
|
||||
{
|
||||
id: 'a5',
|
||||
fromBlockId: 'aristotle',
|
||||
toBlockId: 'ferpa',
|
||||
label: 'obeys',
|
||||
kind: 'constraintApplies',
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'a6',
|
||||
fromBlockId: 'student',
|
||||
toBlockId: 'course',
|
||||
label: 'enrolled_in',
|
||||
kind: 'association',
|
||||
confidence: 1.0,
|
||||
},
|
||||
],
|
||||
constraints: [
|
||||
{
|
||||
id: 'ferpa',
|
||||
label: 'FERPA boundary',
|
||||
expression: 'tenancy = institutional',
|
||||
appliesTo: ['aristotle'],
|
||||
confidence: 1.0,
|
||||
},
|
||||
],
|
||||
requirements: [
|
||||
{
|
||||
id: 'req-001',
|
||||
tag: 'REQ-001',
|
||||
text: 'Never produces complete solutions to graded problems',
|
||||
relations: [{ kind: 'satisfy', blockId: 'aristotle' }],
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'req-002',
|
||||
tag: 'REQ-002',
|
||||
text: 'Response latency under 1.2s P50',
|
||||
relations: [{ kind: 'satisfy', blockId: 'aristotle' }],
|
||||
confidence: 1.0,
|
||||
},
|
||||
{
|
||||
id: 'req-003',
|
||||
tag: 'REQ-003',
|
||||
text: 'FERPA tenancy — coursework never leaves institutional boundary',
|
||||
relations: [{ kind: 'satisfy', blockId: 'aristotle' }],
|
||||
confidence: 1.0,
|
||||
},
|
||||
],
|
||||
};
|
||||
74
phase-0/src/generate/generate.ts
Normal file
74
phase-0/src/generate/generate.ts
Normal file
@@ -0,0 +1,74 @@
|
||||
// Seed → GeneratedModel via LM Studio.
|
||||
|
||||
import { chatJSON, type Message } from '../llm/messages.js';
|
||||
import { loadPrompt } from '../prompts/load.js';
|
||||
import { GeneratedModelLean, expand, lintModel } from './schema.js';
|
||||
import { generatedModelLeanJsonSchema } from './json-schema.js';
|
||||
import type { SeedPayload, GeneratedModel } from '../types.js';
|
||||
|
||||
export interface GenerateResult {
|
||||
model: GeneratedModel;
|
||||
warnings: string[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export async function generateModel(seed: SeedPayload): Promise<GenerateResult> {
|
||||
const character = await loadPrompt('character');
|
||||
const generate = await loadPrompt('generate');
|
||||
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: `${character}\n\n---\n\n${generate}` },
|
||||
{
|
||||
role: 'user',
|
||||
content:
|
||||
`Generate the SysML-shaped model for this seed. Return only the JSON object.\n\n` +
|
||||
`\`\`\`json\n${JSON.stringify(stripMeta(seed), null, 2)}\n\`\`\``,
|
||||
},
|
||||
];
|
||||
|
||||
const start = Date.now();
|
||||
const temperature = parseFloat(process.env.PHASE0_TEMP_GENERATE ?? '0.3');
|
||||
|
||||
const { value, result } = await chatJSON<unknown>(messages, {
|
||||
temperature,
|
||||
maxTokens: 4096,
|
||||
jsonSchema: {
|
||||
name: 'sysml_model',
|
||||
schema: generatedModelLeanJsonSchema as unknown as Record<string, unknown>,
|
||||
},
|
||||
useSchema: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
|
||||
// Validate against the lean schema. If it fails, throw with details.
|
||||
const parsed = GeneratedModelLean.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
const err = parsed.error;
|
||||
throw new Error(
|
||||
`Model JSON failed schema validation:\n` +
|
||||
err.issues.map(i => ` ${i.path.join('.')}: ${i.message}`).join('\n') +
|
||||
`\n\nRaw response (first 600 chars): ${JSON.stringify(value).slice(0, 600)}`
|
||||
);
|
||||
}
|
||||
|
||||
const expanded = expand(parsed.data);
|
||||
const lint = lintModel(expanded);
|
||||
const durationMs = Date.now() - start;
|
||||
|
||||
return {
|
||||
model: expanded,
|
||||
warnings: lint.warnings,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
/** Strip the `meta` field — testing-only metadata, not for the LLM. */
|
||||
function stripMeta(seed: SeedPayload): Omit<SeedPayload, 'meta'> {
|
||||
const { meta, ...rest } = seed;
|
||||
void meta;
|
||||
return rest;
|
||||
}
|
||||
103
phase-0/src/generate/json-schema.ts
Normal file
103
phase-0/src/generate/json-schema.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
// Hand-written JSON Schema mirroring GeneratedModelLean from schema.ts.
|
||||
//
|
||||
// LM Studio's response_format requires `json_schema` for our model; this is
|
||||
// the schema we hand the server. We mirror the Zod definition exactly so
|
||||
// that constrained generation produces something Zod can also validate.
|
||||
//
|
||||
// Keep this in sync with src/generate/schema.ts.
|
||||
|
||||
export const generatedModelLeanJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
systemOfInterestId: { type: 'string' },
|
||||
blocks: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string', minLength: 1 },
|
||||
kind: { type: 'string', enum: ['system', 'actor', 'block'] },
|
||||
properties: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
name: { type: 'string', minLength: 1 },
|
||||
type: {
|
||||
type: 'object',
|
||||
// Tagged union by `kind`. Any kind is accepted; for `enum` we also
|
||||
// require `values`. The model occasionally adds extra fields; allow
|
||||
// them rather than failing the whole response.
|
||||
properties: {
|
||||
kind: { type: 'string', enum: ['string', 'number', 'boolean', 'enum'] },
|
||||
values: { type: 'array', items: { type: 'string' } },
|
||||
},
|
||||
required: ['kind'],
|
||||
},
|
||||
},
|
||||
required: ['name', 'type'],
|
||||
},
|
||||
},
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
},
|
||||
required: ['id', 'label', 'kind', 'confidence'],
|
||||
},
|
||||
},
|
||||
associations: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
fromBlockId: { type: 'string', minLength: 1 },
|
||||
toBlockId: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string' },
|
||||
kind: {
|
||||
type: 'string',
|
||||
enum: ['association', 'composition', 'aggregation', 'generalization', 'constraintApplies'],
|
||||
},
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
},
|
||||
required: ['id', 'fromBlockId', 'toBlockId', 'kind', 'confidence'],
|
||||
},
|
||||
},
|
||||
constraints: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string', minLength: 1 },
|
||||
expression: { type: 'string' },
|
||||
appliesTo: { type: 'array', items: { type: 'string' } },
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
},
|
||||
required: ['id', 'label', 'confidence'],
|
||||
},
|
||||
},
|
||||
requirements: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
tag: { type: 'string', minLength: 1 },
|
||||
text: { type: 'string', minLength: 1 },
|
||||
satisfiedBy: { type: 'array', items: { type: 'string' } },
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
},
|
||||
required: ['id', 'tag', 'text', 'confidence'],
|
||||
},
|
||||
},
|
||||
overallConfidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
notes: { type: 'string' },
|
||||
},
|
||||
required: ['blocks', 'overallConfidence'],
|
||||
} as const;
|
||||
206
phase-0/src/generate/schema.ts
Normal file
206
phase-0/src/generate/schema.ts
Normal file
@@ -0,0 +1,206 @@
|
||||
// Schema for the seed→model generation step.
|
||||
//
|
||||
// Strategy: ask the LLM for a *lean* shape (smaller, easier for 4B-class models),
|
||||
// validate it with Zod, then expand into the full GeneratedModel that downstream
|
||||
// modules expect (with multiplicities, stereotypes, property ids, etc. filled in).
|
||||
//
|
||||
// This separation is defensive: small models often emit mostly-correct JSON.
|
||||
// The lean schema gives them less to get wrong; the expand step is deterministic.
|
||||
|
||||
import { z } from 'zod';
|
||||
import type {
|
||||
GeneratedModel,
|
||||
GeneratedBlock,
|
||||
GeneratedAssociation,
|
||||
GeneratedConstraint,
|
||||
GeneratedRequirement,
|
||||
Property,
|
||||
PropertyType,
|
||||
} from '../types.js';
|
||||
|
||||
// ─── Lean LLM-facing shape ────────────────────────────────────────────────
|
||||
|
||||
const PropertyTypeLean = z.discriminatedUnion('kind', [
|
||||
z.object({ kind: z.literal('string') }),
|
||||
z.object({ kind: z.literal('number') }),
|
||||
z.object({ kind: z.literal('boolean') }),
|
||||
z.object({ kind: z.literal('enum'), values: z.array(z.string()).min(1) }),
|
||||
]);
|
||||
|
||||
const PropertyLean = z.object({
|
||||
name: z.string().min(1),
|
||||
type: PropertyTypeLean,
|
||||
});
|
||||
|
||||
const BlockLean = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
kind: z.enum(['system', 'actor', 'block']),
|
||||
properties: z.array(PropertyLean).default([]),
|
||||
// Tolerate slightly out-of-range values (small models occasionally emit > 1 or < 0)
|
||||
// by clamping rather than rejecting. The whole batch failing because of one
|
||||
// stray 1.05 is the wrong tradeoff at Phase 0.
|
||||
confidence: z.number().transform(n => Math.max(0, Math.min(1, n))),
|
||||
});
|
||||
|
||||
const AssociationLean = z.object({
|
||||
id: z.string().min(1),
|
||||
fromBlockId: z.string().min(1),
|
||||
toBlockId: z.string().min(1),
|
||||
label: z.string().default(''),
|
||||
kind: z.enum(['association', 'composition', 'aggregation', 'generalization', 'constraintApplies']),
|
||||
// Tolerate slightly out-of-range values (small models occasionally emit > 1 or < 0)
|
||||
// by clamping rather than rejecting. The whole batch failing because of one
|
||||
// stray 1.05 is the wrong tradeoff at Phase 0.
|
||||
confidence: z.number().transform(n => Math.max(0, Math.min(1, n))),
|
||||
});
|
||||
|
||||
const ConstraintLean = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
expression: z.string().default(''),
|
||||
appliesTo: z.array(z.string()).default([]),
|
||||
// Tolerate slightly out-of-range values (small models occasionally emit > 1 or < 0)
|
||||
// by clamping rather than rejecting. The whole batch failing because of one
|
||||
// stray 1.05 is the wrong tradeoff at Phase 0.
|
||||
confidence: z.number().transform(n => Math.max(0, Math.min(1, n))),
|
||||
});
|
||||
|
||||
const RequirementLean = z.object({
|
||||
id: z.string().min(1),
|
||||
tag: z.string().min(1),
|
||||
text: z.string().min(1),
|
||||
satisfiedBy: z.array(z.string()).default([]),
|
||||
// Tolerate slightly out-of-range values (small models occasionally emit > 1 or < 0)
|
||||
// by clamping rather than rejecting. The whole batch failing because of one
|
||||
// stray 1.05 is the wrong tradeoff at Phase 0.
|
||||
confidence: z.number().transform(n => Math.max(0, Math.min(1, n))),
|
||||
});
|
||||
|
||||
export const GeneratedModelLean = z.object({
|
||||
systemOfInterestId: z.string().optional(),
|
||||
blocks: z.array(BlockLean),
|
||||
associations: z.array(AssociationLean).default([]),
|
||||
constraints: z.array(ConstraintLean).default([]),
|
||||
requirements: z.array(RequirementLean).default([]),
|
||||
overallConfidence: z.number().min(0).max(1),
|
||||
notes: z.string().optional(),
|
||||
});
|
||||
|
||||
export type GeneratedModelLean = z.infer<typeof GeneratedModelLean>;
|
||||
|
||||
// ─── Expand lean → full GeneratedModel ───────────────────────────────────
|
||||
|
||||
export function expand(lean: GeneratedModelLean): GeneratedModel {
|
||||
// Pick the SoI: trust the field, else find the (single) kind:'system' block,
|
||||
// else leave undefined.
|
||||
let soiId = lean.systemOfInterestId;
|
||||
if (!soiId) {
|
||||
const systems = lean.blocks.filter(b => b.kind === 'system');
|
||||
if (systems.length === 1) soiId = systems[0]!.id;
|
||||
}
|
||||
|
||||
const blocks: GeneratedBlock[] = lean.blocks.map(b => ({
|
||||
id: b.id,
|
||||
label: b.label,
|
||||
kind: b.kind as GeneratedBlock['kind'],
|
||||
stereotypes: [b.kind],
|
||||
properties: b.properties.map((p, i) => expandProperty(p, b.id, i)),
|
||||
confidence: b.confidence,
|
||||
}));
|
||||
|
||||
const associations: GeneratedAssociation[] = lean.associations.map(a => ({
|
||||
id: a.id,
|
||||
fromBlockId: a.fromBlockId,
|
||||
toBlockId: a.toBlockId,
|
||||
label: a.label,
|
||||
kind: a.kind,
|
||||
confidence: a.confidence,
|
||||
}));
|
||||
|
||||
const constraints: GeneratedConstraint[] = lean.constraints.map(c => ({
|
||||
id: c.id,
|
||||
label: c.label,
|
||||
expression: c.expression,
|
||||
appliesTo: c.appliesTo,
|
||||
confidence: c.confidence,
|
||||
}));
|
||||
|
||||
const requirements: GeneratedRequirement[] = lean.requirements.map(r => ({
|
||||
id: r.id,
|
||||
tag: r.tag,
|
||||
text: r.text,
|
||||
relations: r.satisfiedBy.map(blockId => ({ kind: 'satisfy' as const, blockId })),
|
||||
confidence: r.confidence,
|
||||
}));
|
||||
|
||||
return {
|
||||
systemOfInterestId: soiId,
|
||||
blocks,
|
||||
associations,
|
||||
constraints,
|
||||
requirements,
|
||||
overallConfidence: lean.overallConfidence,
|
||||
notes: lean.notes,
|
||||
};
|
||||
}
|
||||
|
||||
function expandProperty(
|
||||
p: z.infer<typeof PropertyLean>,
|
||||
blockId: string,
|
||||
index: number
|
||||
): Property {
|
||||
return {
|
||||
id: `${blockId}_p${index + 1}`,
|
||||
name: p.name,
|
||||
type: p.type as PropertyType,
|
||||
multiplicity: '0..1',
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Sanity checks (post-validation, before rendering) ───────────────────
|
||||
|
||||
export interface ModelLint {
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export function lintModel(model: GeneratedModel): ModelLint {
|
||||
const warnings: string[] = [];
|
||||
const blockIds = new Set(model.blocks.map(b => b.id));
|
||||
|
||||
// System count
|
||||
const systems = model.blocks.filter(b => b.kind === 'system');
|
||||
if (systems.length === 0) warnings.push('No block has kind="system" — missing System of Interest');
|
||||
if (systems.length > 1) warnings.push(`Multiple system blocks: ${systems.map(s => s.id).join(', ')}`);
|
||||
|
||||
// Association endpoints
|
||||
for (const a of model.associations) {
|
||||
if (!blockIds.has(a.fromBlockId)) warnings.push(`Association ${a.id} from unknown block: ${a.fromBlockId}`);
|
||||
if (!blockIds.has(a.toBlockId)) warnings.push(`Association ${a.id} to unknown block: ${a.toBlockId}`);
|
||||
}
|
||||
|
||||
// Constraint applications
|
||||
for (const c of model.constraints) {
|
||||
for (const target of c.appliesTo) {
|
||||
if (!blockIds.has(target)) warnings.push(`Constraint ${c.id} applies to unknown block: ${target}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Requirement satisfiers
|
||||
for (const r of model.requirements) {
|
||||
for (const rel of r.relations) {
|
||||
if (rel.kind === 'satisfy' && !blockIds.has(rel.blockId)) {
|
||||
warnings.push(`Requirement ${r.tag} satisfied by unknown block: ${rel.blockId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Duplicate ids
|
||||
const idCounts = new Map<string, number>();
|
||||
for (const b of model.blocks) idCounts.set(b.id, (idCounts.get(b.id) ?? 0) + 1);
|
||||
for (const [id, count] of idCounts) {
|
||||
if (count > 1) warnings.push(`Duplicate block id: ${id} (×${count})`);
|
||||
}
|
||||
|
||||
return { warnings };
|
||||
}
|
||||
27
phase-0/src/llm/client.ts
Normal file
27
phase-0/src/llm/client.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
// LM Studio client — OpenAI SDK pointed at the local endpoint.
|
||||
import OpenAI from 'openai';
|
||||
import 'dotenv/config';
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
// Load .env.local first if present (gitignored secrets), then .env (defaults)
|
||||
const envLocal = resolve(process.cwd(), '.env.local');
|
||||
if (existsSync(envLocal)) loadEnv({ path: envLocal, override: true });
|
||||
|
||||
const baseURL = process.env.LMSTUDIO_BASE_URL ?? 'http://localhost:1234/v1';
|
||||
const apiKey = process.env.LMSTUDIO_API_KEY ?? 'lm-studio';
|
||||
export const modelName = process.env.LMSTUDIO_MODEL ?? 'local-model';
|
||||
|
||||
export const client = new OpenAI({
|
||||
baseURL,
|
||||
apiKey,
|
||||
// LM Studio's local endpoint is fine with default fetch; long generations
|
||||
// can take a while on smaller hardware so we bump the timeout.
|
||||
timeout: 120_000,
|
||||
maxRetries: 0, // we handle retries ourselves in messages.ts
|
||||
});
|
||||
|
||||
export function describeClient(): string {
|
||||
return `${modelName} @ ${baseURL}`;
|
||||
}
|
||||
143
phase-0/src/llm/messages.ts
Normal file
143
phase-0/src/llm/messages.ts
Normal file
@@ -0,0 +1,143 @@
|
||||
// Chat helper — wraps the LM Studio client with retries and JSON-mode handling.
|
||||
//
|
||||
// Small local models often struggle with strict JSON-schema mode. Strategy:
|
||||
// 1. Try the call as requested.
|
||||
// 2. If JSON parsing fails, retry up to N times with the validation error
|
||||
// appended to the message history ("repair" pattern).
|
||||
// 3. On other failures (network, 5xx), exponential backoff retry.
|
||||
|
||||
import { client, modelName } from './client.js';
|
||||
import type { ChatCompletion } from 'openai/resources/chat/completions';
|
||||
|
||||
export type Role = 'system' | 'user' | 'assistant';
|
||||
export interface Message {
|
||||
role: Role;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ChatOptions {
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
/** Pass a JSON schema (Zod-derived) to constrain output. */
|
||||
jsonSchema?: { name: string; schema: Record<string, unknown> };
|
||||
/** Force JSON-object mode (looser than schema). */
|
||||
jsonObjectMode?: boolean;
|
||||
}
|
||||
|
||||
export interface ChatResult {
|
||||
text: string;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
raw: ChatCompletion;
|
||||
}
|
||||
|
||||
export async function chat(
|
||||
messages: Message[],
|
||||
opts: ChatOptions = {}
|
||||
): Promise<ChatResult> {
|
||||
const responseFormat = buildResponseFormat(opts);
|
||||
let lastErr: unknown;
|
||||
const maxAttempts = 3;
|
||||
|
||||
if (process.env.PHASE0_DEBUG === '1') {
|
||||
const charCount = messages.reduce((sum, m) => sum + m.content.length, 0);
|
||||
console.error(`[debug] chat call: ${messages.length} msgs, ~${charCount} chars (~${Math.round(charCount / 4)} tokens), maxTokens=${opts.maxTokens ?? 4096}`);
|
||||
}
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
const completion = await client.chat.completions.create({
|
||||
model: modelName,
|
||||
messages,
|
||||
temperature: opts.temperature ?? 0.3,
|
||||
max_tokens: opts.maxTokens ?? 4096,
|
||||
...(responseFormat ? { response_format: responseFormat as never } : {}),
|
||||
});
|
||||
|
||||
const choice = completion.choices[0];
|
||||
const text = choice?.message?.content ?? '';
|
||||
return {
|
||||
text,
|
||||
inputTokens: completion.usage?.prompt_tokens ?? 0,
|
||||
outputTokens: completion.usage?.completion_tokens ?? 0,
|
||||
raw: completion,
|
||||
};
|
||||
} catch (err) {
|
||||
lastErr = err;
|
||||
if (attempt < maxAttempts) {
|
||||
const backoff = 500 * 2 ** (attempt - 1);
|
||||
await new Promise(r => setTimeout(r, backoff));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastErr ?? new Error('chat failed without an explicit error');
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JSON from a chat response with repair-retry.
|
||||
*
|
||||
* If the first parse fails, re-asks the model with the parse error appended,
|
||||
* up to `maxRepairs` times.
|
||||
*/
|
||||
export async function chatJSON<T = unknown>(
|
||||
messages: Message[],
|
||||
opts: ChatOptions & { maxRepairs?: number; useSchema?: boolean } = {}
|
||||
): Promise<{ value: T; result: ChatResult }> {
|
||||
const { maxRepairs = 2, useSchema = false, ...rest } = opts;
|
||||
const callOpts: ChatOptions = useSchema && opts.jsonSchema
|
||||
? { ...rest, jsonSchema: opts.jsonSchema }
|
||||
: { ...rest, jsonObjectMode: true };
|
||||
|
||||
const history: Message[] = [...messages];
|
||||
|
||||
for (let attempt = 0; attempt <= maxRepairs; attempt++) {
|
||||
const result = await chat(history, callOpts);
|
||||
const cleaned = stripCodeFences(result.text);
|
||||
try {
|
||||
const value = JSON.parse(cleaned) as T;
|
||||
return { value, result };
|
||||
} catch (parseErr) {
|
||||
if (attempt === maxRepairs) {
|
||||
throw new Error(
|
||||
`JSON parse failed after ${maxRepairs + 1} attempts. ` +
|
||||
`Last error: ${(parseErr as Error).message}\n` +
|
||||
`Last response (first 500 chars): ${cleaned.slice(0, 500)}`
|
||||
);
|
||||
}
|
||||
// Add the broken response and a repair instruction to the history.
|
||||
history.push({ role: 'assistant', content: result.text });
|
||||
history.push({
|
||||
role: 'user',
|
||||
content:
|
||||
`That response was not valid JSON. Error: ${(parseErr as Error).message}\n` +
|
||||
`Return ONLY a valid JSON object, no prose, no code fences.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable, but TypeScript doesn't know that.
|
||||
throw new Error('chatJSON exhausted repairs');
|
||||
}
|
||||
|
||||
function buildResponseFormat(opts: ChatOptions): unknown {
|
||||
if (opts.jsonSchema) {
|
||||
return {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: opts.jsonSchema.name,
|
||||
schema: opts.jsonSchema.schema,
|
||||
strict: false, // small models don't honor strict; use as a hint
|
||||
},
|
||||
};
|
||||
}
|
||||
if (opts.jsonObjectMode) return { type: 'json_object' };
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function stripCodeFences(text: string): string {
|
||||
// Some models wrap JSON in ```json ... ``` fences despite instructions.
|
||||
const trimmed = text.trim();
|
||||
const fenceMatch = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/);
|
||||
if (fenceMatch) return fenceMatch[1].trim();
|
||||
return trimmed;
|
||||
}
|
||||
25
phase-0/src/prompts/character.md
Normal file
25
phase-0/src/prompts/character.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# Socrates
|
||||
|
||||
You are Socrates, a thinking partner for a product manager designing a product idea inside Socrata. You speak with peerage — not as an assistant, as a colleague.
|
||||
|
||||
## Voice
|
||||
- Question-led. Default to surfacing the right question rather than volunteering a solution.
|
||||
- Economical. Sentences carry weight. No filler.
|
||||
- Skeptical by default. Neutral or mildly contrarian, never optimistic.
|
||||
- Concrete. Refer to specific model elements by name when possible.
|
||||
- Decisive when threads run long. After 2–3 iterations on a point, recommend.
|
||||
|
||||
## Never
|
||||
- Open with affirmations like "Great question" or "Sure".
|
||||
- Recap what the user just said before responding.
|
||||
- Apologize for limitations.
|
||||
- Ask permission to draft when you could just propose.
|
||||
- Cheerlead a weak idea.
|
||||
- Use bullet points for prose responses.
|
||||
- Reference any element not in the current model.
|
||||
|
||||
## Pattern
|
||||
When responding, follow this structure unless the user asked a direct factual question:
|
||||
1. Observe what just happened or what's true now.
|
||||
2. Name the underlying tension or implication.
|
||||
3. Propose a concrete next move (with numbered options if a decision is wanted).
|
||||
50
phase-0/src/prompts/detect-assumptions.md
Normal file
50
phase-0/src/prompts/detect-assumptions.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# Detect implicit assumptions in a product seed and model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: surface the **implicit assumptions** the user is making — beliefs treated as true without explicit validation.
|
||||
|
||||
## What is an assumption
|
||||
|
||||
A measurable, falsifiable belief that underpins the idea but isn't stated as a requirement or constraint. Examples:
|
||||
|
||||
- "Students will accept a tool that refuses to answer" — assumes adoption willingness
|
||||
- "1.2s P50 latency is achievable on-prem with available models" — assumes technical feasibility
|
||||
- "Faculty will not classify Socratic prompts as academic dishonesty" — assumes institutional acceptance
|
||||
|
||||
## What is NOT an assumption
|
||||
|
||||
- Stated requirements (REQ-NNN entries) — those are explicit goals
|
||||
- Constraints — those are non-negotiables, not beliefs
|
||||
- Definitions of terms
|
||||
- Generic startup truisms ("users will want this") — too vague to be a useful assumption
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings` — an array of assumption candidates. Each candidate:
|
||||
|
||||
- `text` — the assumption restated cleanly, in one sentence, in the user's register
|
||||
- `linkedElementIds` — array of model element ids this assumption is about (block ids, requirement ids, or constraint ids — must match what's in the model)
|
||||
- `confidence` — 0.0 to 1.0, how confident you are this is genuinely an unstated assumption
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.5`
|
||||
- Cap at 8 findings
|
||||
- Each assumption must name a SPECIFIC, falsifiable belief — not a generic concern
|
||||
- Each must reference at least one real element id from the model
|
||||
- If the seed is sparse and you cannot surface real assumptions, return fewer (or none) rather than fabricating
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
45
phase-0/src/prompts/detect-inconsistencies.md
Normal file
45
phase-0/src/prompts/detect-inconsistencies.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# Detect inconsistencies in a generated model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: find **inconsistencies** — internal contradictions or structural problems in the model.
|
||||
|
||||
## Categories
|
||||
|
||||
- **Internal contradictions** — two requirements that can't both hold simultaneously; a block whose properties contradict its kind; a constraint already violated by some property value
|
||||
- **Reference issues** — an association whose endpoints don't make semantic sense (e.g., actor → constraint, or system → external actor with the wrong direction)
|
||||
- **Over-broad claims** — a requirement that promises more than the system can deliver based on the blocks present
|
||||
- **Missing satisfiers** — a requirement with no plausible block to satisfy it
|
||||
- **Unused elements** — a block with no associations and no requirement satisfaction (may be dead)
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings`. Each candidate:
|
||||
|
||||
- `text` — the inconsistency stated clearly in one sentence
|
||||
- `linkedElementIds` — array of element ids involved
|
||||
- `confidence` — 0.0 to 1.0
|
||||
- `validationCode` — optional. If the issue matches a structural rule, include the code: `M2` (cyclic composition), `T1` (untraced requirement), `T2` (unused element), `S1` (dangling association endpoint). Otherwise omit.
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.6` — for inconsistencies, false positives are worse than misses
|
||||
- Cap at 6 findings
|
||||
- An inconsistency must point to a SPECIFIC contradiction or structural defect, not a stylistic preference
|
||||
- "This block has too many properties" is NOT an inconsistency
|
||||
- "Requirement REQ-002 forbids what association A2 enables" IS an inconsistency
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"confidence": 0.0,
|
||||
"validationCode": "string (optional)"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
46
phase-0/src/prompts/detect-risks.md
Normal file
46
phase-0/src/prompts/detect-risks.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# Detect risks in a product seed and model
|
||||
|
||||
You will receive a seed payload (JSON) and a generated model (JSON). Your job: surface **risks** — specific failure modes that could prevent the system from working as intended.
|
||||
|
||||
## Risk categories
|
||||
|
||||
- **Technical** — feasibility, performance, scaling
|
||||
- **Market** — adoption, competitive dynamics, distribution
|
||||
- **Execution** — team capability, timing, dependencies
|
||||
- **Regulatory** — compliance, legal, privacy
|
||||
- **External** — third-party reliance, geopolitical, supply
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with a single field `findings` — an array of risk candidates. Each candidate:
|
||||
|
||||
- `text` — the risk restated as a specific failure mode in one sentence
|
||||
- `linkedElementIds` — array of model element ids this risk implicates
|
||||
- `severity` — `"low"`, `"medium"`, or `"high"` (impact-if-it-happens, NOT probability)
|
||||
- `confidence` — 0.0 to 1.0, how confident you are this is a real risk worth tracking
|
||||
|
||||
## Rules
|
||||
|
||||
- Return only candidates with `confidence ≥ 0.5`
|
||||
- Cap at 6 findings
|
||||
- A risk must name a SPECIFIC failure mode tied to SPECIFIC element(s). "Won't work" is not a risk; "Latency target unachievable on consumer-grade hardware given 7B-param inference" is.
|
||||
- Severity reflects what happens IF the risk materializes, not how likely it is.
|
||||
- Each finding must reference at least one real element id from the model
|
||||
- For vague seeds with weak models, return fewer findings rather than fabricated ones
|
||||
|
||||
## Schema
|
||||
|
||||
```json
|
||||
{
|
||||
"findings": [
|
||||
{
|
||||
"text": "string",
|
||||
"linkedElementIds": ["string"],
|
||||
"severity": "low" | "medium" | "high",
|
||||
"confidence": 0.0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose, no code fences.
|
||||
120
phase-0/src/prompts/generate.md
Normal file
120
phase-0/src/prompts/generate.md
Normal file
@@ -0,0 +1,120 @@
|
||||
# Generate a SysML-shaped product model from a seed idea
|
||||
|
||||
You are an analyst who turns a product manager's seed idea into a structured systems-engineering model.
|
||||
|
||||
## Input
|
||||
You will receive a seed payload as JSON with these fields:
|
||||
- `problem` — the user-named problem (1–3 sentences)
|
||||
- `targetUser` — who experiences the problem
|
||||
- `desiredOutcome` — what success looks like
|
||||
- `initialHypothesis` — optional belief about adoption or mechanism
|
||||
- `constraints` — optional list of explicit non-negotiable rules (literal strings)
|
||||
|
||||
## Output structure — FOUR distinct top-level arrays
|
||||
|
||||
You must populate ALL FOUR of these arrays when the seed supports it. Empty arrays are a strong signal you under-modeled — the seed almost always has at least one of each.
|
||||
|
||||
1. **`blocks`** — entities (kinds: `system`, `actor`, `block`). The thing being built and the things it interacts with or reasons about.
|
||||
2. **`associations`** — labeled relationships between blocks. Verb phrases like `consults`, `enrolled_in`, `scoped_to`.
|
||||
3. **`constraints`** — non-negotiable invariants the system must obey. Each constraint is a SEPARATE entry in the `constraints` array, NOT a block. Example: a regulatory boundary, a hard latency limit, an ethical refusal policy.
|
||||
4. **`requirements`** — tagged statements (REQ-001, REQ-002, …) drawn from the desired outcome and from the seed's explicit `constraints` list. Each requirement lists which block(s) satisfy it.
|
||||
|
||||
## Rules
|
||||
|
||||
**System of Interest (SoI):** Exactly one block has `kind: "system"`. Name it after the *thing being built*, not the problem. For "Aristotle, an AI study companion", the system block is `"Aristotle"`, not `"Disengagement problem"`.
|
||||
|
||||
**Actors:** People or external systems that interact with the SoI. `kind: "actor"`.
|
||||
|
||||
**Blocks:** Things the system reasons about that aren't actors. `kind: "block"`.
|
||||
|
||||
**Constraints (NOT blocks, NOT requirements):** Anything in the seed's `constraints` field, plus any non-negotiable invariant you infer (regulatory, ethical, hard physical limit). Each goes in the `constraints` array with `appliesTo` listing the block ids it constrains. Often `appliesTo` is just the SoI.
|
||||
|
||||
**The Constraint–Requirement boundary (READ THIS):**
|
||||
- A **constraint** is something you **must obey** — non-negotiable, often regulatory or physical. You don't choose to satisfy it; you obey it or you don't ship. Examples: "FERPA tenancy", "hard latency limit", "must never output complete solutions".
|
||||
- A **requirement** is a **goal the system must satisfy** — derived from the desired outcome and from product behavior promises. Examples: "Re-engage students within their first session", "Operate offline for travel use cases".
|
||||
|
||||
**Each item from `seed.constraints` belongs in EXACTLY ONE place — the `constraints` array.** Do NOT also output it as a requirement. If you find yourself authoring REQ-NNN entries that restate the seed's constraints verbatim, stop — those are constraints, not requirements.
|
||||
|
||||
The `requirements` array should contain things derived from `seed.desiredOutcome` and other product-behavior implications — NOT a re-encoding of `seed.constraints`.
|
||||
|
||||
**Associations:**
|
||||
- `association` — generic verb-phrase relationship (default).
|
||||
- `composition` — whole-part. Use ONLY when X is *literally part of* Y.
|
||||
- `generalization` — is-a. Rarely needed for product ideas.
|
||||
- `constraintApplies` — links a constraint to the block(s) it constrains. ONLY use this if you also want a visible edge in the diagram; otherwise rely on the `appliesTo` field of the constraint itself.
|
||||
|
||||
**Requirements:** Each gets a tag like `REQ-001`. Each must list `satisfiedBy` — a non-empty array of block ids that fulfill it. **Derive requirements from `seed.desiredOutcome`, not from `seed.constraints`** (constraints have their own array). Aim for 1–4 requirements unless the seed clearly demands more.
|
||||
|
||||
**Vague desired-outcome rule:** If `seed.desiredOutcome` is too vague to derive specific requirements (e.g., "Something useful for them", "Make it good", or any single-clause platitude with no measurable criterion), leave the `requirements` array EMPTY. Do NOT invent a placeholder requirement — that's worse than no requirement. The same vagueness signal should drive `overallConfidence` below 0.3.
|
||||
|
||||
**Properties:** A block's properties are its *attributes the system reasons about*. Keep to 1–4 per block. Types: `string`, `number`, `boolean`, or `enum` (with `values`).
|
||||
|
||||
## Confidence — under-suggest rather than over-suggest
|
||||
|
||||
Per element, set a `confidence` in `[0, 1]`:
|
||||
- Seed's explicit nouns → high confidence (≥ 0.85)
|
||||
- Inferred-but-clearly-implied → medium (0.5–0.8)
|
||||
- Speculative → low (< 0.5) and **generally omit**
|
||||
|
||||
A clean, sparse, correct model beats a dense fabricated one. If the seed is too vague to model, return a sparse model and set `overallConfidence` below 0.3.
|
||||
|
||||
## ID conventions
|
||||
|
||||
- Block ids: lowercase snake_case from labels. `"Aristotle"` → `"aristotle"`. `"Coursework Material"` → `"coursework_material"`.
|
||||
- Association ids: `a1`, `a2`, `a3`, …
|
||||
- Constraint ids: lowercase snake_case from labels. `"FERPA boundary"` → `"ferpa_boundary"`.
|
||||
- Requirement ids: lowercase tag with hyphen replaced. `REQ-001` → `"req_001"`.
|
||||
|
||||
## Worked example
|
||||
|
||||
Given a seed about a personal recipe scrapbook that pulls from cooking blogs:
|
||||
|
||||
```json
|
||||
{
|
||||
"systemOfInterestId": "scrapbook",
|
||||
"blocks": [
|
||||
{ "id": "scrapbook", "label": "Scrapbook", "kind": "system",
|
||||
"properties": [
|
||||
{ "name": "private_collection", "type": { "kind": "boolean" } }
|
||||
],
|
||||
"confidence": 0.95 },
|
||||
{ "id": "home_cook", "label": "Home Cook", "kind": "actor",
|
||||
"properties": [
|
||||
{ "name": "skill_level", "type": { "kind": "enum", "values": ["beginner","intermediate","expert"] } }
|
||||
],
|
||||
"confidence": 0.95 },
|
||||
{ "id": "cooking_blog", "label": "Cooking Blog", "kind": "actor",
|
||||
"properties": [],
|
||||
"confidence": 0.9 },
|
||||
{ "id": "recipe", "label": "Recipe", "kind": "block",
|
||||
"properties": [
|
||||
{ "name": "ingredients", "type": { "kind": "string" } },
|
||||
{ "name": "steps", "type": { "kind": "string" } }
|
||||
],
|
||||
"confidence": 1.0 }
|
||||
],
|
||||
"associations": [
|
||||
{ "id": "a1", "fromBlockId": "home_cook", "toBlockId": "scrapbook", "label": "uses", "kind": "association", "confidence": 0.95 },
|
||||
{ "id": "a2", "fromBlockId": "scrapbook", "toBlockId": "cooking_blog", "label": "imports_from", "kind": "association", "confidence": 0.9 },
|
||||
{ "id": "a3", "fromBlockId": "scrapbook", "toBlockId": "recipe", "label": "contains", "kind": "composition", "confidence": 1.0 }
|
||||
],
|
||||
"constraints": [
|
||||
{ "id": "copyright_respect", "label": "Copyright respect", "expression": "must not republish recipes outside the user's private collection",
|
||||
"appliesTo": ["scrapbook"], "confidence": 0.85 }
|
||||
],
|
||||
"requirements": [
|
||||
{ "id": "req_001", "tag": "REQ-001", "text": "Imports a recipe from a URL in under 5 seconds",
|
||||
"satisfiedBy": ["scrapbook"], "confidence": 0.9 },
|
||||
{ "id": "req_002", "tag": "REQ-002", "text": "Stores recipes in the user's private collection only",
|
||||
"satisfiedBy": ["scrapbook"], "confidence": 1.0 }
|
||||
],
|
||||
"overallConfidence": 0.85,
|
||||
"notes": "The Scrapbook is the SoI; home cook and cooking blog are actors; recipes are first-class blocks."
|
||||
}
|
||||
```
|
||||
|
||||
Notice every array is populated. No constraints in `blocks`. Requirements name specific block satisfiers.
|
||||
|
||||
## Now generate
|
||||
|
||||
Return ONLY the JSON object for the seed you receive. No prose, no code fences. Use the four arrays — fill all of them.
|
||||
67
phase-0/src/prompts/judge.md
Normal file
67
phase-0/src/prompts/judge.md
Normal file
@@ -0,0 +1,67 @@
|
||||
# Score a Phase 0 run against the rubric
|
||||
|
||||
You are an evaluator. You receive a complete Phase 0 run for one seed: the seed payload, the LLM-generated SysML model, and the detected findings (assumptions, risks, inconsistencies).
|
||||
|
||||
Your job: score the run on **seven dimensions**, each on a 1–5 integer scale, with a one-sentence justification per dimension.
|
||||
|
||||
You are NOT the model author. You are scoring an LLM's output. Apply extra scrutiny — small models drift toward sycophancy and over-population.
|
||||
|
||||
## Scale
|
||||
|
||||
- **5** — A senior PM colleague would approve this without changes.
|
||||
- **4** — Good; minor edits would tighten it.
|
||||
- **3** — Useful but flawed; would need real iteration before sharing.
|
||||
- **2** — Wrong in significant ways; would mislead a PM relying on it.
|
||||
- **1** — Misleading or hallucinated; worse than no model at all.
|
||||
|
||||
## The seven dimensions
|
||||
|
||||
1. **modelCoverage** — Did the model identify the major entities a real PM thinking about this seed would name? Penalize missing actors, missing the System of Interest's core dependents.
|
||||
2. **modelAccuracy** — Are the relationships correct? Compositions actually whole-part? Generalizations actually is-a? Penalize miscategorized association kinds.
|
||||
3. **modelParsimony** — Is the model uncluttered, or padded with fabricated entities? Penalize "filler" blocks added to look thorough.
|
||||
4. **constraintCapture** — Are the meaningful non-functional constraints surfaced? Both explicit (from `seed.constraints`) and inferred. Penalize duplicating the same constraint as both a Constraint and a Requirement.
|
||||
5. **assumptionDetectionQuality** — Are the assumptions specific, falsifiable, and tied to real elements — or generic startup truisms? Penalize "users will adopt this" type platitudes.
|
||||
6. **riskDetectionQuality** — Are the risks substantive failure modes, domain-specific, tied to elements? Penalize generic risks ("competition exists", "scaling is hard").
|
||||
7. **voiceAndCharacter** — Across the prose visible (notes, finding texts), does Socrates sound: question-led? skeptical? economical? concrete? Penalize hedging, sycophancy, and bullet-style prose where prose was expected.
|
||||
|
||||
## Failure-prone seeds — score against what the seed enables, not absolutes
|
||||
|
||||
For seeds tagged `expectedDifficulty: failure-prone`, score every "did the model produce X" dimension **relative to what the seed could plausibly support**, not against an absolute completeness target. A vague seed has nothing to derive constraints or rich coverage from — refusing to fabricate is the correct behavior.
|
||||
|
||||
Specifically:
|
||||
|
||||
- **modelCoverage** — if the seed names few entities, a sparse model is correct. Penalize only if the model missed entities the seed explicitly mentions.
|
||||
- **modelAccuracy** — judge structural correctness of what's there, not what's absent.
|
||||
- **modelParsimony** — sparse + low confidence on a vague seed = 5/5.
|
||||
- **constraintCapture** — if the seed has NO `constraints` field (or it's empty), 0 model constraints is 5/5, not 1/5. Penalize only if the seed has explicit constraints the model failed to capture.
|
||||
- **assumptionDetectionQuality** — generic "will be adopted" platitudes still score low; assumptions that name the seed's vagueness as the underlying problem score HIGH.
|
||||
- **riskDetectionQuality** — risks that flag the seed's vagueness itself score HIGH.
|
||||
- **voiceAndCharacter** — always evaluated against the character spec; no seed-difficulty inversion.
|
||||
|
||||
The principle: **for a vague seed, the right answer is to push back, not to fill in the blanks.** A model that fabricates entities to fill a vague seed scores LOW on parsimony AND coverage.
|
||||
|
||||
## Output schema
|
||||
|
||||
```json
|
||||
{
|
||||
"modelCoverage": 1,
|
||||
"modelCoverageReason": "string",
|
||||
"modelAccuracy": 1,
|
||||
"modelAccuracyReason": "string",
|
||||
"modelParsimony": 1,
|
||||
"modelParsimonyReason": "string",
|
||||
"constraintCapture": 1,
|
||||
"constraintCaptureReason": "string",
|
||||
"assumptionDetectionQuality": 1,
|
||||
"assumptionDetectionQualityReason": "string",
|
||||
"riskDetectionQuality": 1,
|
||||
"riskDetectionQualityReason": "string",
|
||||
"voiceAndCharacter": 1,
|
||||
"voiceAndCharacterReason": "string",
|
||||
"overallComment": "string (1–2 sentences, optional)"
|
||||
}
|
||||
```
|
||||
|
||||
All seven scores are required integers in [1, 5]. Reasons are required strings, ≤30 words each. `overallComment` is optional.
|
||||
|
||||
Return ONLY the JSON object. No prose preamble, no code fences.
|
||||
16
phase-0/src/prompts/load.ts
Normal file
16
phase-0/src/prompts/load.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
// Load prompt files from src/prompts/ as plain strings.
|
||||
//
|
||||
// Prompts are .md files so they can be edited without recompiling. They live
|
||||
// alongside the .ts files so tsx + tsc (which both resolve relative paths
|
||||
// from src/) can find them at the same path in dev and production.
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export async function loadPrompt(name: string): Promise<string> {
|
||||
const path = resolve(here, `${name}.md`);
|
||||
return readFile(path, 'utf8');
|
||||
}
|
||||
44
phase-0/src/prompts/propose.md
Normal file
44
phase-0/src/prompts/propose.md
Normal file
@@ -0,0 +1,44 @@
|
||||
# Mode: Propose a model change
|
||||
|
||||
You are reviewing the existing model and the active findings (assumptions, risks, inconsistencies). The PM has asked you to propose ONE concrete change to the model.
|
||||
|
||||
Pick the highest-value change you can make. Examples:
|
||||
- Remove a fabricated or unused element
|
||||
- Add a missing actor or block that the seed clearly implies but the model didn't capture
|
||||
- Add a missing constraint
|
||||
- Add a missing requirement linked to specific blocks
|
||||
- Remove a duplicate (e.g., a constraint already covered by a requirement, or vice versa)
|
||||
- Add an association the model is missing
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with:
|
||||
|
||||
- `reasoning` — 1–3 sentences explaining what you propose to change and why. Reference specific element ids.
|
||||
- `ops` — an array of operations. Each op is one of:
|
||||
- `{ "kind": "add-block", "block": { id, label, kind: 'system'|'actor'|'block', properties: [{name, type:{kind, values?}}] } }`
|
||||
- `{ "kind": "remove-block", "blockId": "id" }` (also removes dependent associations and references)
|
||||
- `{ "kind": "add-association", "association": { id, fromBlockId, toBlockId, label, kind: 'association'|'composition'|'generalization' } }`
|
||||
- `{ "kind": "remove-association", "associationId": "id" }`
|
||||
- `{ "kind": "add-constraint", "constraint": { id, label, expression, appliesTo: ["blockId"] } }`
|
||||
- `{ "kind": "remove-constraint", "constraintId": "id" }`
|
||||
- `{ "kind": "add-requirement", "requirement": { id, tag, text, satisfiedBy: ["blockId"] } }`
|
||||
- `{ "kind": "remove-requirement", "requirementId": "id" }`
|
||||
|
||||
## Rules
|
||||
|
||||
- Prefer SMALL changes. 1–4 ops is ideal. A proposal that rewrites half the model is too big.
|
||||
- Do not propose cosmetic changes (label tweaks, position changes — those are auto-applied).
|
||||
- Every id you reference (`blockId`, `associationId`, etc.) must exist in the current model — except for ids you're creating in this same proposal.
|
||||
- If you can't see a high-value change worth proposing, return an empty `ops` array and explain why in `reasoning`.
|
||||
|
||||
## Output schema
|
||||
|
||||
```json
|
||||
{
|
||||
"reasoning": "string (1–3 sentences)",
|
||||
"ops": [ /* array of op objects per the kinds above */ ]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose preamble, no code fences.
|
||||
48
phase-0/src/prompts/review.md
Normal file
48
phase-0/src/prompts/review.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Mode: Review (active conversation about an existing model)
|
||||
|
||||
You are mid-project with a PM. The model already exists. You have just been shown the seed, the model, and the active findings (assumptions, risks, inconsistencies).
|
||||
|
||||
Your job in this mode is to surface the most important question or tension and engage the PM in deciding what to do about it. Stay in character per the system prompt.
|
||||
|
||||
## Output
|
||||
|
||||
Return a JSON object with these fields:
|
||||
|
||||
- `text` — your turn, in prose. 1–4 sentences. Follow the Observe → Name tension → Propose pattern from the character spec. No bullets.
|
||||
- `options` (optional, max 3) — when offering a decision, supply numbered options. Each option:
|
||||
- `n` — 1, 2, or 3
|
||||
- `label` — ≤5 words, the choice
|
||||
- `sub` — ≤8 words, a one-line clarifier
|
||||
|
||||
## When to use options
|
||||
|
||||
- The user is at a decision point and continued open prose will spiral
|
||||
- Two or three credible directions exist and you want to make them visible
|
||||
|
||||
## When NOT to use options
|
||||
|
||||
- The user is exploring or just answered a question — let them think
|
||||
- Only one good answer exists — give it, don't pretend
|
||||
- Already-listed options just got declined
|
||||
|
||||
## What never to do
|
||||
|
||||
- Open with "Great question" or "Sure"
|
||||
- Recap what the user said
|
||||
- Apologize for limitations
|
||||
- Cheerlead
|
||||
- Use bullets in the prose `text` field
|
||||
- Reference any element id not in the model JSON shown to you
|
||||
|
||||
## Output schema (strict)
|
||||
|
||||
```json
|
||||
{
|
||||
"text": "string (1–4 sentences)",
|
||||
"options": [
|
||||
{ "n": 1, "label": "string", "sub": "string" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Return ONLY the JSON object. No prose preamble, no code fences.
|
||||
32
phase-0/src/render/render.ts
Normal file
32
phase-0/src/render/render.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
// Render PlantUML source to PNG via the PlantUML server.
|
||||
//
|
||||
// Uses the `plantuml-encoder` package's deflate-base64 encoding; the public
|
||||
// server at https://www.plantuml.com/plantuml accepts it at /png/<encoded>.
|
||||
|
||||
import { encode } from 'plantuml-encoder';
|
||||
import { writeFile } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
|
||||
const SERVER = process.env.PLANTUML_SERVER ?? 'https://www.plantuml.com/plantuml';
|
||||
|
||||
export async function renderPNG(plantumlSource: string, outPath: string): Promise<void> {
|
||||
const encoded = encode(plantumlSource);
|
||||
const url = `${SERVER}/png/${encoded}`;
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`PlantUML server returned ${res.status}: ${res.statusText}`);
|
||||
}
|
||||
const buf = Buffer.from(await res.arrayBuffer());
|
||||
await mkdir(dirname(outPath), { recursive: true });
|
||||
await writeFile(outPath, buf);
|
||||
}
|
||||
|
||||
export async function writePlantUMLSource(source: string, outPath: string): Promise<void> {
|
||||
await mkdir(dirname(outPath), { recursive: true });
|
||||
await writeFile(outPath, source, 'utf8');
|
||||
}
|
||||
|
||||
export function plantUMLViewerURL(plantumlSource: string): string {
|
||||
return `${SERVER}/uml/${encode(plantumlSource)}`;
|
||||
}
|
||||
158
phase-0/src/render/to-plantuml.ts
Normal file
158
phase-0/src/render/to-plantuml.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
// GeneratedModel → PlantUML class-diagram source string.
|
||||
//
|
||||
// We render SysML-shaped class diagrams using stereotypes. PlantUML's
|
||||
// auto-layout is mediocre; we accept that — Phase 0 evaluates content,
|
||||
// not visuals.
|
||||
|
||||
import type {
|
||||
GeneratedModel,
|
||||
GeneratedBlock,
|
||||
GeneratedAssociation,
|
||||
GeneratedConstraint,
|
||||
GeneratedRequirement,
|
||||
AssociationKind,
|
||||
} from '../types.js';
|
||||
|
||||
const SKINPARAMS = `
|
||||
skinparam backgroundColor #f5efe2
|
||||
skinparam shadowing false
|
||||
skinparam class {
|
||||
BackgroundColor #fdfaf0
|
||||
BorderColor #b8a982
|
||||
ArrowColor #6e5d3d
|
||||
FontName "Helvetica"
|
||||
}
|
||||
skinparam note {
|
||||
BackgroundColor #f0e3c4
|
||||
BorderColor #b8a982
|
||||
}
|
||||
hide empty members
|
||||
`.trim();
|
||||
|
||||
export function toPlantUML(model: GeneratedModel): string {
|
||||
const lines: string[] = [];
|
||||
lines.push('@startuml');
|
||||
lines.push(SKINPARAMS);
|
||||
lines.push('');
|
||||
|
||||
// Blocks
|
||||
for (const block of model.blocks) {
|
||||
lines.push(renderBlock(block, model.systemOfInterestId));
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Constraints (rendered as classes with «constraint» stereotype, dashed border)
|
||||
for (const constraint of model.constraints) {
|
||||
lines.push(renderConstraint(constraint));
|
||||
}
|
||||
if (model.constraints.length > 0) lines.push('');
|
||||
|
||||
// Associations
|
||||
for (const assoc of model.associations) {
|
||||
lines.push(renderAssociation(assoc));
|
||||
}
|
||||
// Synthesized constraint-applies edges (dashed) from constraint.appliesTo.
|
||||
// Only emitted for constraint→block pairs not already covered by an explicit association.
|
||||
const explicitPairs = new Set(
|
||||
model.associations.map(a => `${a.fromBlockId}→${a.toBlockId}`)
|
||||
);
|
||||
for (const c of model.constraints) {
|
||||
for (const targetId of c.appliesTo) {
|
||||
const key = `${c.id}→${targetId}`;
|
||||
if (!explicitPairs.has(key)) {
|
||||
lines.push(`${sanitizeId(c.id)} ..> ${sanitizeId(targetId)} : applies_to`);
|
||||
}
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Requirements as a single floating note
|
||||
if (model.requirements.length > 0) {
|
||||
lines.push(renderRequirementsNote(model.requirements));
|
||||
}
|
||||
|
||||
lines.push('@enduml');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function renderBlock(block: GeneratedBlock, soiId: string | undefined): string {
|
||||
const stereotype = stereoLabel(block.kind);
|
||||
const classKeyword = block.kind === 'system' || block.id === soiId ? 'class' : 'class';
|
||||
const safeId = sanitizeId(block.id);
|
||||
const props = block.properties
|
||||
.map(p => ` ${escapeText(p.name)} : ${typeLabel(p.type)}`)
|
||||
.join('\n');
|
||||
const body = props ? ` {\n${props}\n}` : '';
|
||||
// The system block gets a slightly heavier border via a #color hint.
|
||||
const borderHint = block.kind === 'system' ? ' #2c4a6b' : '';
|
||||
return `${classKeyword} "${escapeText(block.label)}" as ${safeId} <<${stereotype}>>${borderHint}${body}`;
|
||||
}
|
||||
|
||||
function renderConstraint(c: GeneratedConstraint): string {
|
||||
const safeId = sanitizeId(c.id);
|
||||
const expr = c.expression ? `\n ${escapeText(c.expression)}` : '';
|
||||
return `class "${escapeText(c.label)}" as ${safeId} <<constraint>> {${expr}\n}`;
|
||||
}
|
||||
|
||||
function renderAssociation(a: GeneratedAssociation): string {
|
||||
const from = sanitizeId(a.fromBlockId);
|
||||
const to = sanitizeId(a.toBlockId);
|
||||
const arrow = arrowFor(a.kind);
|
||||
const label = a.label ? ` : ${escapeText(a.label)}` : '';
|
||||
return `${from} ${arrow} ${to}${label}`;
|
||||
}
|
||||
|
||||
function renderRequirementsNote(reqs: GeneratedRequirement[]): string {
|
||||
const lines = reqs.map(r => ` **${r.tag}** — ${escapeText(r.text)}`);
|
||||
return ['legend right', ' **Requirements**', ...lines, 'endlegend'].join('\n');
|
||||
}
|
||||
|
||||
function stereoLabel(kind: GeneratedBlock['kind']): string {
|
||||
switch (kind) {
|
||||
case 'block':
|
||||
return 'block';
|
||||
case 'actor':
|
||||
return 'actor';
|
||||
case 'constraint':
|
||||
return 'constraint';
|
||||
case 'system':
|
||||
return 'system';
|
||||
}
|
||||
}
|
||||
|
||||
function arrowFor(kind: AssociationKind): string {
|
||||
switch (kind) {
|
||||
case 'association':
|
||||
return '-->';
|
||||
case 'composition':
|
||||
return '*--';
|
||||
case 'aggregation':
|
||||
return 'o--';
|
||||
case 'generalization':
|
||||
return '<|--';
|
||||
case 'constraintApplies':
|
||||
return '..>';
|
||||
}
|
||||
}
|
||||
|
||||
function typeLabel(type: GeneratedBlock['properties'][number]['type']): string {
|
||||
switch (type.kind) {
|
||||
case 'string':
|
||||
return 'string';
|
||||
case 'number':
|
||||
return 'number';
|
||||
case 'boolean':
|
||||
return 'boolean';
|
||||
case 'enum':
|
||||
return `enum{${type.values.join(',')}}`;
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeId(id: string): string {
|
||||
// PlantUML class names must be alphanumeric (with underscores).
|
||||
return id.replace(/[^A-Za-z0-9_]/g, '_');
|
||||
}
|
||||
|
||||
function escapeText(text: string): string {
|
||||
return text.replace(/"/g, '\\"').replace(/\n/g, ' ');
|
||||
}
|
||||
52
phase-0/src/seed/load.ts
Normal file
52
phase-0/src/seed/load.ts
Normal file
@@ -0,0 +1,52 @@
|
||||
// Load seed JSON files.
|
||||
|
||||
import { readFile, readdir } from 'node:fs/promises';
|
||||
import { resolve, basename } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
import type { SeedPayload } from '../types.js';
|
||||
|
||||
const SeedSchema = z.object({
|
||||
id: z.string(),
|
||||
title: z.string(),
|
||||
problem: z.string(),
|
||||
targetUser: z.string(),
|
||||
desiredOutcome: z.string(),
|
||||
initialHypothesis: z.string().optional(),
|
||||
constraints: z.array(z.string()).optional(),
|
||||
notes: z.string().optional(),
|
||||
meta: z.object({
|
||||
isHoldout: z.boolean(),
|
||||
expectedDifficulty: z.enum(['easy', 'medium', 'hard', 'failure-prone']),
|
||||
testsFor: z.array(z.string()),
|
||||
}),
|
||||
});
|
||||
|
||||
const seedsDir = resolve(process.cwd(), 'seeds');
|
||||
|
||||
export async function loadSeed(id: string): Promise<SeedPayload> {
|
||||
const path = resolve(seedsDir, `${id}.json`);
|
||||
const raw = JSON.parse(await readFile(path, 'utf8'));
|
||||
return SeedSchema.parse(raw);
|
||||
}
|
||||
|
||||
export async function listSeedIds(): Promise<string[]> {
|
||||
const files = await readdir(seedsDir);
|
||||
return files
|
||||
.filter(f => f.endsWith('.json'))
|
||||
.map(f => basename(f, '.json'))
|
||||
.sort();
|
||||
}
|
||||
|
||||
export async function loadCorpus(opts: { includeHoldouts: boolean }): Promise<SeedPayload[]> {
|
||||
const ids = await listSeedIds();
|
||||
const all = await Promise.all(ids.map(loadSeed));
|
||||
if (opts.includeHoldouts) {
|
||||
if (process.env.PHASE0_FINAL_ROUND !== '1') {
|
||||
throw new Error(
|
||||
'Holdouts can only be loaded when PHASE0_FINAL_ROUND=1. Set the env var to acknowledge final-round usage.'
|
||||
);
|
||||
}
|
||||
return all;
|
||||
}
|
||||
return all.filter(s => !s.meta.isHoldout);
|
||||
}
|
||||
302
phase-0/src/socrates/converse.ts
Normal file
302
phase-0/src/socrates/converse.ts
Normal file
@@ -0,0 +1,302 @@
|
||||
// Interactive Socrates conversation loop.
|
||||
//
|
||||
// Runs in the terminal: shows the model + findings as opening context,
|
||||
// then alternates user input ↔ Socrates turn until the user types `:done`,
|
||||
// hits Ctrl+D, or 5 user turns are reached.
|
||||
//
|
||||
// Saves the transcript as outputs/<seedId>/conversation.md.
|
||||
|
||||
import { z } from 'zod';
|
||||
import * as readline from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import { writeFile, mkdir } from 'node:fs/promises';
|
||||
import { dirname } from 'node:path';
|
||||
import kleur from 'kleur';
|
||||
import { chatJSON, type Message } from '../llm/messages.js';
|
||||
import { loadPrompt } from '../prompts/load.js';
|
||||
import type {
|
||||
ConversationTurn,
|
||||
Finding,
|
||||
GeneratedModel,
|
||||
SeedPayload,
|
||||
} from '../types.js';
|
||||
|
||||
// ─── Schema for Socrates' structured turn ────────────────────────────────
|
||||
|
||||
const SocratesTurn = z.object({
|
||||
text: z.string().min(1),
|
||||
options: z
|
||||
.array(
|
||||
z.object({
|
||||
n: z.number().int().min(1).max(3),
|
||||
label: z.string().min(1).max(60),
|
||||
sub: z.string().max(80).optional(),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
});
|
||||
|
||||
const socratesTurnJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['text'],
|
||||
properties: {
|
||||
text: { type: 'string', minLength: 1 },
|
||||
options: {
|
||||
type: 'array',
|
||||
maxItems: 3,
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['n', 'label'],
|
||||
properties: {
|
||||
n: { type: 'integer', minimum: 1, maximum: 3 },
|
||||
label: { type: 'string', minLength: 1, maxLength: 60 },
|
||||
sub: { type: 'string', maxLength: 80 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ConverseOptions {
|
||||
outPath: string;
|
||||
maxUserTurns?: number;
|
||||
}
|
||||
|
||||
export interface ConverseResult {
|
||||
turns: ConversationTurn[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export async function runConversation(
|
||||
seed: SeedPayload,
|
||||
model: GeneratedModel,
|
||||
findings: Finding[],
|
||||
opts: ConverseOptions
|
||||
): Promise<ConverseResult> {
|
||||
const character = await loadPrompt('character');
|
||||
const reviewPrompt = await loadPrompt('review');
|
||||
const maxTurns = opts.maxUserTurns ?? 5;
|
||||
|
||||
const systemPrompt = buildSystemPrompt(character, reviewPrompt, seed, model, findings);
|
||||
|
||||
const turns: ConversationTurn[] = [];
|
||||
let totalIn = 0;
|
||||
let totalOut = 0;
|
||||
const start = Date.now();
|
||||
|
||||
// Open with a Socrates turn — opens the conversation cleanly.
|
||||
const opening = await getSocratesTurn(systemPrompt, turns);
|
||||
totalIn += opening.inputTokens;
|
||||
totalOut += opening.outputTokens;
|
||||
pushSocratesTurn(turns, opening.turn);
|
||||
printSocrates(opening.turn);
|
||||
|
||||
const rl = readline.createInterface({ input, output });
|
||||
|
||||
try {
|
||||
for (let userTurnNum = 0; userTurnNum < maxTurns; userTurnNum++) {
|
||||
let userText: string;
|
||||
try {
|
||||
const remaining = maxTurns - userTurnNum;
|
||||
const promptLine = kleur.bold(kleur.cyan(`\nyou (${remaining} turn${remaining === 1 ? '' : 's'} left, :done to stop) > `));
|
||||
userText = (await rl.question(promptLine)).trim();
|
||||
} catch {
|
||||
// Ctrl+D
|
||||
break;
|
||||
}
|
||||
|
||||
if (!userText || userText === ':done' || userText === ':q') break;
|
||||
|
||||
// If user typed a single digit and Socrates' last turn had options, expand it.
|
||||
const lastSocrates = [...turns].reverse().find(t => t.who === 'socrates');
|
||||
const expanded = expandShortcutInput(userText, lastSocrates);
|
||||
|
||||
turns.push({ who: 'user', text: expanded, ts: new Date().toISOString() });
|
||||
|
||||
const next = await getSocratesTurn(systemPrompt, turns);
|
||||
totalIn += next.inputTokens;
|
||||
totalOut += next.outputTokens;
|
||||
pushSocratesTurn(turns, next.turn);
|
||||
printSocrates(next.turn);
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
|
||||
const transcript = renderTranscript(seed, turns);
|
||||
await mkdir(dirname(opts.outPath), { recursive: true });
|
||||
await writeFile(opts.outPath, transcript, 'utf8');
|
||||
|
||||
return {
|
||||
turns,
|
||||
inputTokens: totalIn,
|
||||
outputTokens: totalOut,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Internals ───────────────────────────────────────────────────────────
|
||||
|
||||
function buildSystemPrompt(
|
||||
character: string,
|
||||
reviewPrompt: string,
|
||||
seed: SeedPayload,
|
||||
model: GeneratedModel,
|
||||
findings: Finding[]
|
||||
): string {
|
||||
const trimmedModel = trimModel(model);
|
||||
const trimmedFindings = findings.map(f => ({
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds,
|
||||
...(f.severity ? { severity: f.severity } : {}),
|
||||
...(f.validationCode ? { validationCode: f.validationCode } : {}),
|
||||
}));
|
||||
|
||||
return [
|
||||
character,
|
||||
'---',
|
||||
reviewPrompt,
|
||||
'---',
|
||||
'Project context:',
|
||||
'```json',
|
||||
JSON.stringify({ seed: stripMeta(seed), model: trimmedModel, findings: trimmedFindings }, null, 2),
|
||||
'```',
|
||||
].join('\n\n');
|
||||
}
|
||||
|
||||
async function getSocratesTurn(
|
||||
systemPrompt: string,
|
||||
history: ConversationTurn[]
|
||||
): Promise<{ turn: ConversationTurn; inputTokens: number; outputTokens: number }> {
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: systemPrompt },
|
||||
...history.map(t => ({
|
||||
role: (t.who === 'socrates' ? 'assistant' : 'user') as 'assistant' | 'user',
|
||||
content: t.text + (t.options ? `\n${formatOptionsAsText(t.options)}` : ''),
|
||||
})),
|
||||
];
|
||||
|
||||
// Nudge the assistant if there's no last user turn (opening case)
|
||||
if (history.length === 0) {
|
||||
messages.push({
|
||||
role: 'user',
|
||||
content: 'Open the conversation. Surface the most important tension you see in this model.',
|
||||
});
|
||||
}
|
||||
|
||||
const temperature = parseFloat(process.env.PHASE0_TEMP_REVIEW ?? '0.4');
|
||||
|
||||
const { value, result } = await chatJSON<unknown>(messages, {
|
||||
temperature,
|
||||
maxTokens: 768,
|
||||
jsonSchema: { name: 'socrates_turn', schema: socratesTurnJsonSchema as Record<string, unknown> },
|
||||
useSchema: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
|
||||
const parsed = SocratesTurn.safeParse(value);
|
||||
const turn: ConversationTurn = parsed.success
|
||||
? {
|
||||
who: 'socrates',
|
||||
text: parsed.data.text,
|
||||
options: parsed.data.options,
|
||||
ts: new Date().toISOString(),
|
||||
}
|
||||
: {
|
||||
who: 'socrates',
|
||||
text: '[Socrates produced an unparseable turn — skipping]',
|
||||
ts: new Date().toISOString(),
|
||||
};
|
||||
|
||||
return {
|
||||
turn,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function pushSocratesTurn(turns: ConversationTurn[], turn: ConversationTurn): void {
|
||||
turns.push(turn);
|
||||
}
|
||||
|
||||
function printSocrates(turn: ConversationTurn): void {
|
||||
console.log();
|
||||
console.log(kleur.bold('Σ Socrates') + kleur.dim(` · ${shortTime(turn.ts)}`));
|
||||
console.log(turn.text);
|
||||
if (turn.options) {
|
||||
console.log();
|
||||
for (const o of turn.options) {
|
||||
const num = kleur.bold(kleur.cyan(`${o.n}.`));
|
||||
const sub = o.sub ? kleur.dim(` · ${o.sub}`) : '';
|
||||
console.log(` ${num} ${o.label}${sub}`);
|
||||
}
|
||||
console.log(kleur.dim(` press 1–${turn.options.length} or type a reply`));
|
||||
}
|
||||
}
|
||||
|
||||
function expandShortcutInput(text: string, lastSocratesTurn: ConversationTurn | undefined): string {
|
||||
if (!lastSocratesTurn?.options) return text;
|
||||
const m = text.match(/^([1-3])$/);
|
||||
if (!m) return text;
|
||||
const num = parseInt(m[1]!, 10);
|
||||
const opt = lastSocratesTurn.options.find(o => o.n === num);
|
||||
if (!opt) return text;
|
||||
return `[option ${opt.n}] ${opt.label}${opt.sub ? ` — ${opt.sub}` : ''}`;
|
||||
}
|
||||
|
||||
function formatOptionsAsText(options: NonNullable<ConversationTurn['options']>): string {
|
||||
return options.map(o => `${o.n}. ${o.label}${o.sub ? ` · ${o.sub}` : ''}`).join('\n');
|
||||
}
|
||||
|
||||
function trimModel(model: GeneratedModel): unknown {
|
||||
return {
|
||||
systemOfInterestId: model.systemOfInterestId,
|
||||
blocks: model.blocks.map(b => ({ id: b.id, label: b.label, kind: b.kind, properties: b.properties.map(p => p.name) })),
|
||||
associations: model.associations.map(a => ({ id: a.id, from: a.fromBlockId, to: a.toBlockId, label: a.label, kind: a.kind })),
|
||||
constraints: model.constraints.map(c => ({ id: c.id, label: c.label, appliesTo: c.appliesTo })),
|
||||
requirements: model.requirements.map(r => ({ id: r.id, tag: r.tag, text: r.text })),
|
||||
};
|
||||
}
|
||||
|
||||
function stripMeta(seed: SeedPayload): Omit<SeedPayload, 'meta'> {
|
||||
const { meta, ...rest } = seed;
|
||||
void meta;
|
||||
return rest;
|
||||
}
|
||||
|
||||
function shortTime(iso: string): string {
|
||||
const d = new Date(iso);
|
||||
return d.toTimeString().slice(0, 8);
|
||||
}
|
||||
|
||||
function renderTranscript(seed: SeedPayload, turns: ConversationTurn[]): string {
|
||||
const header = [
|
||||
`# Socrates conversation — ${seed.title}`,
|
||||
``,
|
||||
`_Seed: ${seed.id} · ${turns.length} turns · ${new Date().toISOString()}_`,
|
||||
``,
|
||||
`---`,
|
||||
``,
|
||||
].join('\n');
|
||||
|
||||
const body = turns
|
||||
.map(t => {
|
||||
if (t.who === 'socrates') {
|
||||
const optsBlock = t.options
|
||||
? '\n\n' + t.options.map(o => `- **${o.n}.** ${o.label}${o.sub ? ` — _${o.sub}_` : ''}`).join('\n')
|
||||
: '';
|
||||
return `### Σ Socrates\n\n${t.text}${optsBlock}`;
|
||||
}
|
||||
return `### You\n\n${t.text}`;
|
||||
})
|
||||
.join('\n\n');
|
||||
|
||||
return header + body + '\n';
|
||||
}
|
||||
267
phase-0/src/socrates/detect.ts
Normal file
267
phase-0/src/socrates/detect.ts
Normal file
@@ -0,0 +1,267 @@
|
||||
// Detection — runs all three detection prompts (assumptions, risks, inconsistencies)
|
||||
// against a generated model and returns a flat list of findings.
|
||||
|
||||
import { z } from 'zod';
|
||||
import { chatJSON, type Message } from '../llm/messages.js';
|
||||
import { loadPrompt } from '../prompts/load.js';
|
||||
import { postValidateFindings } from './post-validate.js';
|
||||
import type { GeneratedModel, SeedPayload, Finding } from '../types.js';
|
||||
|
||||
// ─── Schemas for each detection mode ─────────────────────────────────────
|
||||
|
||||
const AssumptionFindings = z.object({
|
||||
findings: z.array(z.object({
|
||||
text: z.string().min(1),
|
||||
linkedElementIds: z.array(z.string()).optional(),
|
||||
confidence: z.number().min(0).max(1),
|
||||
})).optional(),
|
||||
});
|
||||
|
||||
const RiskFindings = z.object({
|
||||
findings: z.array(z.object({
|
||||
text: z.string().min(1),
|
||||
linkedElementIds: z.array(z.string()).optional(),
|
||||
confidence: z.number().min(0).max(1),
|
||||
severity: z.enum(['low', 'medium', 'high']),
|
||||
})).optional(),
|
||||
});
|
||||
|
||||
const InconsistencyFindings = z.object({
|
||||
findings: z.array(z.object({
|
||||
text: z.string().min(1),
|
||||
linkedElementIds: z.array(z.string()).optional(),
|
||||
confidence: z.number().min(0).max(1),
|
||||
validationCode: z.string().optional(),
|
||||
})).optional(),
|
||||
});
|
||||
|
||||
// JSON Schemas for LM Studio's response_format
|
||||
const baseFindingProperties = {
|
||||
text: { type: 'string', minLength: 1 },
|
||||
linkedElementIds: { type: 'array', items: { type: 'string' } },
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
};
|
||||
|
||||
const assumptionsJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['findings'],
|
||||
properties: {
|
||||
findings: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['text', 'confidence'],
|
||||
properties: baseFindingProperties,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const risksJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['findings'],
|
||||
properties: {
|
||||
findings: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['text', 'confidence', 'severity'],
|
||||
properties: {
|
||||
...baseFindingProperties,
|
||||
severity: { type: 'string', enum: ['low', 'medium', 'high'] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const inconsistenciesJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['findings'],
|
||||
properties: {
|
||||
findings: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['text', 'confidence'],
|
||||
properties: {
|
||||
...baseFindingProperties,
|
||||
validationCode: { type: 'string' },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface DetectResult {
|
||||
findings: Finding[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
strippedRefs: number;
|
||||
droppedFindings: number;
|
||||
}
|
||||
|
||||
export async function detectFindings(
|
||||
model: GeneratedModel,
|
||||
seed: SeedPayload
|
||||
): Promise<DetectResult> {
|
||||
const character = await loadPrompt('character');
|
||||
|
||||
const start = Date.now();
|
||||
let totalIn = 0;
|
||||
let totalOut = 0;
|
||||
|
||||
const userPayload =
|
||||
`Seed:\n\`\`\`json\n${JSON.stringify(stripMeta(seed), null, 2)}\n\`\`\`\n\n` +
|
||||
`Model:\n\`\`\`json\n${JSON.stringify(stripModel(model), null, 2)}\n\`\`\`\n`;
|
||||
|
||||
// Parallel detection — three independent passes share the same context
|
||||
// window now that LM Studio has been bumped. The character prompt is
|
||||
// included to keep the voice consistent across the system.
|
||||
const [assumptions, risks, inconsistencies] = await Promise.all([
|
||||
runDetect(character, 'detect-assumptions', userPayload, assumptionsJsonSchema, AssumptionFindings),
|
||||
runDetect(character, 'detect-risks', userPayload, risksJsonSchema, RiskFindings),
|
||||
runDetect(character, 'detect-inconsistencies', userPayload, inconsistenciesJsonSchema, InconsistencyFindings),
|
||||
]);
|
||||
|
||||
for (const r of [assumptions, risks, inconsistencies]) {
|
||||
totalIn += r.inputTokens;
|
||||
totalOut += r.outputTokens;
|
||||
}
|
||||
|
||||
const rawFindings: Finding[] = [
|
||||
...(assumptions.parsed?.findings ?? []).map(f => ({
|
||||
kind: 'assumption' as const,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: f.confidence,
|
||||
})),
|
||||
...(risks.parsed?.findings ?? []).map(f => ({
|
||||
kind: 'risk' as const,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: f.confidence,
|
||||
severity: f.severity,
|
||||
})),
|
||||
...(inconsistencies.parsed?.findings ?? []).map(f => ({
|
||||
kind: 'inconsistency' as const,
|
||||
text: f.text,
|
||||
linkedElementIds: f.linkedElementIds ?? [],
|
||||
confidence: f.confidence,
|
||||
validationCode: f.validationCode,
|
||||
})),
|
||||
];
|
||||
|
||||
// Strip hallucinated element references — see post-validate.ts.
|
||||
const validated = postValidateFindings(rawFindings, model);
|
||||
|
||||
return {
|
||||
findings: validated.findings,
|
||||
inputTokens: totalIn,
|
||||
outputTokens: totalOut,
|
||||
durationMs: Date.now() - start,
|
||||
strippedRefs: validated.strippedRefs.length,
|
||||
droppedFindings: validated.droppedFindings,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Internals ───────────────────────────────────────────────────────────
|
||||
|
||||
interface PassResult<TParsed> {
|
||||
parsed: TParsed | null;
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
}
|
||||
|
||||
async function runDetect<TSchema extends z.ZodTypeAny>(
|
||||
character: string,
|
||||
promptName: string,
|
||||
userPayload: string,
|
||||
jsonSchema: object,
|
||||
responseSchema: TSchema
|
||||
): Promise<PassResult<z.infer<TSchema>>> {
|
||||
const prompt = await loadPrompt(promptName);
|
||||
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: `${character}\n\n---\n\n${prompt}` },
|
||||
{ role: 'user', content: userPayload },
|
||||
];
|
||||
|
||||
const temperature = parseFloat(process.env.PHASE0_TEMP_DETECT ?? '0.2');
|
||||
|
||||
const { value, result } = await chatJSON<unknown>(messages, {
|
||||
temperature,
|
||||
maxTokens: 1536,
|
||||
jsonSchema: { name: 'findings', schema: jsonSchema as Record<string, unknown> },
|
||||
useSchema: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
|
||||
const parsed = responseSchema.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
console.error(
|
||||
`[${promptName}] schema validation failed; returning no findings. ` +
|
||||
`Errors: ${parsed.error.issues.map(i => `${i.path.join('.')}: ${i.message}`).join('; ')}`
|
||||
);
|
||||
return { parsed: null, inputTokens: result.inputTokens, outputTokens: result.outputTokens };
|
||||
}
|
||||
|
||||
return {
|
||||
parsed: parsed.data,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function stripMeta(seed: SeedPayload): Omit<SeedPayload, 'meta'> {
|
||||
const { meta, ...rest } = seed;
|
||||
void meta;
|
||||
return rest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact, detection-friendly view of the model. Drops all property metadata
|
||||
* (multiplicity, type ids), confidence scores, and stereotypes — the detector
|
||||
* only needs ids, labels, kinds, and the relationship/constraint/requirement
|
||||
* structure. Cuts payload size by ~70%.
|
||||
*/
|
||||
function stripModel(model: GeneratedModel): unknown {
|
||||
return {
|
||||
systemOfInterestId: model.systemOfInterestId,
|
||||
blocks: model.blocks.map(b => ({
|
||||
id: b.id,
|
||||
label: b.label,
|
||||
kind: b.kind,
|
||||
properties: b.properties.map(p => p.name),
|
||||
})),
|
||||
associations: model.associations.map(a => ({
|
||||
id: a.id,
|
||||
from: a.fromBlockId,
|
||||
to: a.toBlockId,
|
||||
label: a.label,
|
||||
kind: a.kind,
|
||||
})),
|
||||
constraints: model.constraints.map(c => ({
|
||||
id: c.id,
|
||||
label: c.label,
|
||||
appliesTo: c.appliesTo,
|
||||
})),
|
||||
requirements: model.requirements.map(r => ({
|
||||
id: r.id,
|
||||
tag: r.tag,
|
||||
text: r.text,
|
||||
satisfiedBy: r.relations
|
||||
.filter(rel => rel.kind === 'satisfy')
|
||||
.map(rel => (rel as { kind: 'satisfy'; blockId: string }).blockId),
|
||||
})),
|
||||
};
|
||||
}
|
||||
68
phase-0/src/socrates/post-validate.ts
Normal file
68
phase-0/src/socrates/post-validate.ts
Normal file
@@ -0,0 +1,68 @@
|
||||
// Post-validation for findings — strips hallucinated element references.
|
||||
//
|
||||
// Detection prompts can name element ids that don't exist in the live model
|
||||
// (seen ~5% of the time on Aristotle with a 4B model). We can't reliably
|
||||
// catch this in the prompt; we filter it here so downstream code can trust
|
||||
// every linkedElementId resolves.
|
||||
//
|
||||
// Mirrors the production guard specified in docs/socrates.md §6.3.
|
||||
|
||||
import type { Finding, GeneratedModel } from '../types.js';
|
||||
|
||||
export interface PostValidateResult {
|
||||
findings: Finding[];
|
||||
strippedRefs: Array<{ findingIndex: number; ref: string; reason: string }>;
|
||||
droppedFindings: number; // findings dropped because every ref was bad
|
||||
}
|
||||
|
||||
export function postValidateFindings(
|
||||
findings: Finding[],
|
||||
model: GeneratedModel
|
||||
): PostValidateResult {
|
||||
const validIds = collectValidIds(model);
|
||||
const strippedRefs: PostValidateResult['strippedRefs'] = [];
|
||||
const cleaned: Finding[] = [];
|
||||
let dropped = 0;
|
||||
|
||||
findings.forEach((finding, idx) => {
|
||||
const goodRefs: string[] = [];
|
||||
for (const ref of finding.linkedElementIds) {
|
||||
if (validIds.has(ref)) {
|
||||
goodRefs.push(ref);
|
||||
} else {
|
||||
strippedRefs.push({ findingIndex: idx, ref, reason: 'not in model' });
|
||||
}
|
||||
}
|
||||
|
||||
// Drop the finding entirely only if EVERY ref was bad AND there were refs to begin with.
|
||||
// A finding with zero refs is suspicious but not necessarily wrong (some inconsistencies
|
||||
// are about absences); we keep them for human review.
|
||||
if (finding.linkedElementIds.length > 0 && goodRefs.length === 0) {
|
||||
dropped++;
|
||||
return;
|
||||
}
|
||||
|
||||
cleaned.push({ ...finding, linkedElementIds: goodRefs });
|
||||
});
|
||||
|
||||
return { findings: cleaned, strippedRefs, droppedFindings: dropped };
|
||||
}
|
||||
|
||||
function collectValidIds(model: GeneratedModel): Set<string> {
|
||||
const ids = new Set<string>();
|
||||
for (const b of model.blocks) ids.add(b.id);
|
||||
for (const a of model.associations) ids.add(a.id);
|
||||
for (const c of model.constraints) ids.add(c.id);
|
||||
for (const r of model.requirements) {
|
||||
ids.add(r.id);
|
||||
ids.add(r.tag); // some prompts may reference REQ-001 instead of req_001
|
||||
}
|
||||
// Property-id form: blockId.propertyName (matches the convention from sysml-modeling.md §8.2)
|
||||
for (const b of model.blocks) {
|
||||
for (const p of b.properties) {
|
||||
ids.add(`${b.id}.${p.name}`);
|
||||
ids.add(p.name); // tolerate bare property names too — common LLM behavior
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
486
phase-0/src/socrates/propose.ts
Normal file
486
phase-0/src/socrates/propose.ts
Normal file
@@ -0,0 +1,486 @@
|
||||
// Socrates proposes a single model change.
|
||||
//
|
||||
// Returns a structured patch (a list of ops) plus reasoning. The CLI applies
|
||||
// the patch deterministically and re-renders. This is the simplest possible
|
||||
// version of the MVP "proposal mediation" flow — Phase 0 doesn't iterate on
|
||||
// proposals; one round, accept or reject.
|
||||
|
||||
import { z } from 'zod';
|
||||
import { chatJSON, type Message } from '../llm/messages.js';
|
||||
import { loadPrompt } from '../prompts/load.js';
|
||||
import type {
|
||||
GeneratedModel,
|
||||
GeneratedBlock,
|
||||
GeneratedAssociation,
|
||||
GeneratedConstraint,
|
||||
GeneratedRequirement,
|
||||
Property,
|
||||
Finding,
|
||||
SeedPayload,
|
||||
} from '../types.js';
|
||||
|
||||
// ─── Op types ────────────────────────────────────────────────────────────
|
||||
|
||||
const AddBlock = z.object({
|
||||
kind: z.literal('add-block'),
|
||||
block: z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
kind: z.enum(['system', 'actor', 'block']),
|
||||
properties: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().min(1),
|
||||
type: z.object({
|
||||
kind: z.enum(['string', 'number', 'boolean', 'enum']),
|
||||
values: z.array(z.string()).optional(),
|
||||
}),
|
||||
})
|
||||
)
|
||||
.optional(),
|
||||
}),
|
||||
});
|
||||
const RemoveBlock = z.object({ kind: z.literal('remove-block'), blockId: z.string().min(1) });
|
||||
const AddAssociation = z.object({
|
||||
kind: z.literal('add-association'),
|
||||
association: z.object({
|
||||
id: z.string().min(1),
|
||||
fromBlockId: z.string().min(1),
|
||||
toBlockId: z.string().min(1),
|
||||
label: z.string().default(''),
|
||||
kind: z.enum(['association', 'composition', 'aggregation', 'generalization']),
|
||||
}),
|
||||
});
|
||||
const RemoveAssociation = z.object({ kind: z.literal('remove-association'), associationId: z.string().min(1) });
|
||||
const AddConstraint = z.object({
|
||||
kind: z.literal('add-constraint'),
|
||||
constraint: z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
expression: z.string().default(''),
|
||||
appliesTo: z.array(z.string()).default([]),
|
||||
}),
|
||||
});
|
||||
const RemoveConstraint = z.object({ kind: z.literal('remove-constraint'), constraintId: z.string().min(1) });
|
||||
const AddRequirement = z.object({
|
||||
kind: z.literal('add-requirement'),
|
||||
requirement: z.object({
|
||||
id: z.string().min(1),
|
||||
tag: z.string().min(1),
|
||||
text: z.string().min(1),
|
||||
satisfiedBy: z.array(z.string()).default([]),
|
||||
}),
|
||||
});
|
||||
const RemoveRequirement = z.object({ kind: z.literal('remove-requirement'), requirementId: z.string().min(1) });
|
||||
|
||||
const Op = z.discriminatedUnion('kind', [
|
||||
AddBlock,
|
||||
RemoveBlock,
|
||||
AddAssociation,
|
||||
RemoveAssociation,
|
||||
AddConstraint,
|
||||
RemoveConstraint,
|
||||
AddRequirement,
|
||||
RemoveRequirement,
|
||||
]);
|
||||
|
||||
const Proposal = z.object({
|
||||
reasoning: z.string().min(1),
|
||||
ops: z.array(Op).default([]),
|
||||
});
|
||||
|
||||
export type ProposalOp = z.infer<typeof Op>;
|
||||
|
||||
// LM Studio JSON schema for response_format.
|
||||
// Op shapes use `oneOf` so the constrained decoder forces the grammar per kind.
|
||||
// A flat tagged-union with open `kind: string` lets small models cram everything
|
||||
// into the kind name (observed on Gemma 4B). oneOf gives the grammar real teeth.
|
||||
|
||||
const propertyTypeSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind'],
|
||||
properties: {
|
||||
kind: { type: 'string', enum: ['string', 'number', 'boolean', 'enum'] },
|
||||
values: { type: 'array', items: { type: 'string' } },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const blockShape = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['id', 'label', 'kind'],
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string', minLength: 1 },
|
||||
kind: { type: 'string', enum: ['system', 'actor', 'block'] },
|
||||
properties: {
|
||||
type: 'array',
|
||||
items: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['name', 'type'],
|
||||
properties: {
|
||||
name: { type: 'string', minLength: 1 },
|
||||
type: propertyTypeSchema,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const associationShape = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['id', 'fromBlockId', 'toBlockId', 'kind'],
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
fromBlockId: { type: 'string', minLength: 1 },
|
||||
toBlockId: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string' },
|
||||
kind: { type: 'string', enum: ['association', 'composition', 'aggregation', 'generalization'] },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const constraintShape = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['id', 'label'],
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
label: { type: 'string', minLength: 1 },
|
||||
expression: { type: 'string' },
|
||||
appliesTo: { type: 'array', items: { type: 'string' } },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const requirementShape = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['id', 'tag', 'text'],
|
||||
properties: {
|
||||
id: { type: 'string', minLength: 1 },
|
||||
tag: { type: 'string', minLength: 1 },
|
||||
text: { type: 'string', minLength: 1 },
|
||||
satisfiedBy: { type: 'array', items: { type: 'string' } },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const proposalJsonSchema = {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['reasoning', 'ops'],
|
||||
properties: {
|
||||
reasoning: { type: 'string', minLength: 1 },
|
||||
ops: {
|
||||
type: 'array',
|
||||
items: {
|
||||
oneOf: [
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'block'],
|
||||
properties: { kind: { type: 'string', enum: ['add-block'] }, block: blockShape },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'blockId'],
|
||||
properties: { kind: { type: 'string', enum: ['remove-block'] }, blockId: { type: 'string', minLength: 1 } },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'association'],
|
||||
properties: { kind: { type: 'string', enum: ['add-association'] }, association: associationShape },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'associationId'],
|
||||
properties: { kind: { type: 'string', enum: ['remove-association'] }, associationId: { type: 'string', minLength: 1 } },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'constraint'],
|
||||
properties: { kind: { type: 'string', enum: ['add-constraint'] }, constraint: constraintShape },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'constraintId'],
|
||||
properties: { kind: { type: 'string', enum: ['remove-constraint'] }, constraintId: { type: 'string', minLength: 1 } },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'requirement'],
|
||||
properties: { kind: { type: 'string', enum: ['add-requirement'] }, requirement: requirementShape },
|
||||
},
|
||||
{
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['kind', 'requirementId'],
|
||||
properties: { kind: { type: 'string', enum: ['remove-requirement'] }, requirementId: { type: 'string', minLength: 1 } },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
// ─── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ProposeResult {
|
||||
reasoning: string;
|
||||
ops: ProposalOp[];
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
export async function proposeChange(
|
||||
seed: SeedPayload,
|
||||
model: GeneratedModel,
|
||||
findings: Finding[]
|
||||
): Promise<ProposeResult> {
|
||||
const character = await loadPrompt('character');
|
||||
const proposePrompt = await loadPrompt('propose');
|
||||
|
||||
const userPayload =
|
||||
`Current model:\n\`\`\`json\n${JSON.stringify(trimModel(model), null, 2)}\n\`\`\`\n\n` +
|
||||
`Active findings:\n\`\`\`json\n${JSON.stringify(trimFindings(findings), null, 2)}\n\`\`\`\n\n` +
|
||||
`Seed:\n\`\`\`json\n${JSON.stringify(stripMeta(seed), null, 2)}\n\`\`\`\n`;
|
||||
|
||||
const messages: Message[] = [
|
||||
{ role: 'system', content: `${character}\n\n---\n\n${proposePrompt}` },
|
||||
{ role: 'user', content: userPayload },
|
||||
];
|
||||
|
||||
const start = Date.now();
|
||||
const { value, result } = await chatJSON<unknown>(messages, {
|
||||
temperature: 0.3,
|
||||
maxTokens: 1024,
|
||||
jsonSchema: { name: 'proposal', schema: proposalJsonSchema as Record<string, unknown> },
|
||||
useSchema: true,
|
||||
maxRepairs: 2,
|
||||
});
|
||||
|
||||
const parsed = Proposal.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
const rawDump = JSON.stringify(value, null, 2).slice(0, 1500);
|
||||
throw new Error(
|
||||
`Proposal JSON failed schema validation:\n` +
|
||||
parsed.error.issues.map(i => ` ${i.path.join('.')}: ${i.message}`).join('\n') +
|
||||
`\n\nRaw response:\n${rawDump}`
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
reasoning: parsed.data.reasoning,
|
||||
ops: parsed.data.ops,
|
||||
inputTokens: result.inputTokens,
|
||||
outputTokens: result.outputTokens,
|
||||
durationMs: Date.now() - start,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Apply ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ApplyResult {
|
||||
model: GeneratedModel;
|
||||
applied: ProposalOp[];
|
||||
skipped: Array<{ op: ProposalOp; reason: string }>;
|
||||
}
|
||||
|
||||
export function applyProposal(model: GeneratedModel, ops: ProposalOp[]): ApplyResult {
|
||||
let next = cloneModel(model);
|
||||
const applied: ProposalOp[] = [];
|
||||
const skipped: ApplyResult['skipped'] = [];
|
||||
|
||||
for (const op of ops) {
|
||||
try {
|
||||
next = applyOne(next, op);
|
||||
applied.push(op);
|
||||
} catch (err) {
|
||||
skipped.push({ op, reason: (err as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
// Recompute the SoI if the prior one was removed.
|
||||
if (next.systemOfInterestId && !next.blocks.find(b => b.id === next.systemOfInterestId)) {
|
||||
const sys = next.blocks.find(b => b.kind === 'system');
|
||||
next = { ...next, systemOfInterestId: sys?.id };
|
||||
}
|
||||
|
||||
return { model: next, applied, skipped };
|
||||
}
|
||||
|
||||
function applyOne(model: GeneratedModel, op: ProposalOp): GeneratedModel {
|
||||
switch (op.kind) {
|
||||
case 'add-block': {
|
||||
if (model.blocks.find(b => b.id === op.block.id)) {
|
||||
throw new Error(`block ${op.block.id} already exists`);
|
||||
}
|
||||
const props: Property[] = (op.block.properties ?? []).map((p, i) => ({
|
||||
id: `${op.block.id}_p${i + 1}`,
|
||||
name: p.name,
|
||||
type: p.type as Property['type'],
|
||||
multiplicity: '0..1',
|
||||
}));
|
||||
const newBlock: GeneratedBlock = {
|
||||
id: op.block.id,
|
||||
label: op.block.label,
|
||||
kind: op.block.kind,
|
||||
stereotypes: [op.block.kind],
|
||||
properties: props,
|
||||
confidence: 0.85,
|
||||
};
|
||||
return { ...model, blocks: [...model.blocks, newBlock] };
|
||||
}
|
||||
case 'remove-block': {
|
||||
if (!model.blocks.find(b => b.id === op.blockId)) {
|
||||
throw new Error(`block ${op.blockId} not found`);
|
||||
}
|
||||
return {
|
||||
...model,
|
||||
blocks: model.blocks.filter(b => b.id !== op.blockId),
|
||||
// cascade: remove associations, constraint applications, requirement satisfiers
|
||||
associations: model.associations.filter(
|
||||
a => a.fromBlockId !== op.blockId && a.toBlockId !== op.blockId
|
||||
),
|
||||
constraints: model.constraints.map(c => ({
|
||||
...c,
|
||||
appliesTo: c.appliesTo.filter(id => id !== op.blockId),
|
||||
})),
|
||||
requirements: model.requirements.map(r => ({
|
||||
...r,
|
||||
relations: r.relations.filter(rel => rel.kind !== 'satisfy' || rel.blockId !== op.blockId),
|
||||
})),
|
||||
};
|
||||
}
|
||||
case 'add-association': {
|
||||
if (model.associations.find(a => a.id === op.association.id)) {
|
||||
throw new Error(`association ${op.association.id} already exists`);
|
||||
}
|
||||
const blockIds = new Set(model.blocks.map(b => b.id));
|
||||
if (!blockIds.has(op.association.fromBlockId)) {
|
||||
throw new Error(`association.fromBlockId ${op.association.fromBlockId} not in model`);
|
||||
}
|
||||
if (!blockIds.has(op.association.toBlockId)) {
|
||||
throw new Error(`association.toBlockId ${op.association.toBlockId} not in model`);
|
||||
}
|
||||
const newAssoc: GeneratedAssociation = {
|
||||
id: op.association.id,
|
||||
fromBlockId: op.association.fromBlockId,
|
||||
toBlockId: op.association.toBlockId,
|
||||
label: op.association.label,
|
||||
kind: op.association.kind,
|
||||
confidence: 0.85,
|
||||
};
|
||||
return { ...model, associations: [...model.associations, newAssoc] };
|
||||
}
|
||||
case 'remove-association': {
|
||||
if (!model.associations.find(a => a.id === op.associationId)) {
|
||||
throw new Error(`association ${op.associationId} not found`);
|
||||
}
|
||||
return { ...model, associations: model.associations.filter(a => a.id !== op.associationId) };
|
||||
}
|
||||
case 'add-constraint': {
|
||||
if (model.constraints.find(c => c.id === op.constraint.id)) {
|
||||
throw new Error(`constraint ${op.constraint.id} already exists`);
|
||||
}
|
||||
const blockIds = new Set(model.blocks.map(b => b.id));
|
||||
const validAppliesTo = op.constraint.appliesTo.filter(id => blockIds.has(id));
|
||||
const newConstraint: GeneratedConstraint = {
|
||||
id: op.constraint.id,
|
||||
label: op.constraint.label,
|
||||
expression: op.constraint.expression,
|
||||
appliesTo: validAppliesTo,
|
||||
confidence: 0.85,
|
||||
};
|
||||
return { ...model, constraints: [...model.constraints, newConstraint] };
|
||||
}
|
||||
case 'remove-constraint': {
|
||||
if (!model.constraints.find(c => c.id === op.constraintId)) {
|
||||
throw new Error(`constraint ${op.constraintId} not found`);
|
||||
}
|
||||
return { ...model, constraints: model.constraints.filter(c => c.id !== op.constraintId) };
|
||||
}
|
||||
case 'add-requirement': {
|
||||
if (model.requirements.find(r => r.id === op.requirement.id)) {
|
||||
throw new Error(`requirement ${op.requirement.id} already exists`);
|
||||
}
|
||||
const blockIds = new Set(model.blocks.map(b => b.id));
|
||||
const newReq: GeneratedRequirement = {
|
||||
id: op.requirement.id,
|
||||
tag: op.requirement.tag,
|
||||
text: op.requirement.text,
|
||||
relations: op.requirement.satisfiedBy
|
||||
.filter(id => blockIds.has(id))
|
||||
.map(id => ({ kind: 'satisfy' as const, blockId: id })),
|
||||
confidence: 0.85,
|
||||
};
|
||||
return { ...model, requirements: [...model.requirements, newReq] };
|
||||
}
|
||||
case 'remove-requirement': {
|
||||
if (!model.requirements.find(r => r.id === op.requirementId)) {
|
||||
throw new Error(`requirement ${op.requirementId} not found`);
|
||||
}
|
||||
return { ...model, requirements: model.requirements.filter(r => r.id !== op.requirementId) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
function cloneModel(model: GeneratedModel): GeneratedModel {
|
||||
return JSON.parse(JSON.stringify(model)) as GeneratedModel;
|
||||
}
|
||||
|
||||
function trimModel(model: GeneratedModel): unknown {
|
||||
return {
|
||||
systemOfInterestId: model.systemOfInterestId,
|
||||
blocks: model.blocks.map(b => ({ id: b.id, label: b.label, kind: b.kind, properties: b.properties.map(p => p.name) })),
|
||||
associations: model.associations.map(a => ({ id: a.id, from: a.fromBlockId, to: a.toBlockId, label: a.label, kind: a.kind })),
|
||||
constraints: model.constraints.map(c => ({ id: c.id, label: c.label, appliesTo: c.appliesTo })),
|
||||
requirements: model.requirements.map(r => ({ id: r.id, tag: r.tag, text: r.text })),
|
||||
};
|
||||
}
|
||||
|
||||
function trimFindings(findings: Finding[]): unknown {
|
||||
return findings.map(f => ({
|
||||
kind: f.kind,
|
||||
text: f.text,
|
||||
refs: f.linkedElementIds,
|
||||
...(f.severity ? { severity: f.severity } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function stripMeta(seed: SeedPayload): Omit<SeedPayload, 'meta'> {
|
||||
const { meta, ...rest } = seed;
|
||||
void meta;
|
||||
return rest;
|
||||
}
|
||||
|
||||
export function summarizeOp(op: ProposalOp): string {
|
||||
switch (op.kind) {
|
||||
case 'add-block':
|
||||
return `+ block "${op.block.label}" (${op.block.kind}) [${op.block.id}]`;
|
||||
case 'remove-block':
|
||||
return `- block ${op.blockId}`;
|
||||
case 'add-association':
|
||||
return `+ assoc ${op.association.fromBlockId} → ${op.association.toBlockId} : ${op.association.label} [${op.association.kind}]`;
|
||||
case 'remove-association':
|
||||
return `- assoc ${op.associationId}`;
|
||||
case 'add-constraint':
|
||||
return `+ constraint "${op.constraint.label}" → ${op.constraint.appliesTo.join(', ') || '(none)'}`;
|
||||
case 'remove-constraint':
|
||||
return `- constraint ${op.constraintId}`;
|
||||
case 'add-requirement':
|
||||
return `+ req ${op.requirement.tag}: ${op.requirement.text} → ${op.requirement.satisfiedBy.join(', ') || '(none)'}`;
|
||||
case 'remove-requirement':
|
||||
return `- req ${op.requirementId}`;
|
||||
}
|
||||
}
|
||||
147
phase-0/src/types.ts
Normal file
147
phase-0/src/types.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
// Shared types for Phase 0 — see docs/phase-0-plan.md §2
|
||||
|
||||
export type BlockKind = 'block' | 'actor' | 'constraint' | 'system';
|
||||
|
||||
export type PropertyType =
|
||||
| { kind: 'string' }
|
||||
| { kind: 'number' }
|
||||
| { kind: 'boolean' }
|
||||
| { kind: 'enum'; values: string[] };
|
||||
|
||||
export type Multiplicity = '0..1' | '1' | '0..*' | '1..*';
|
||||
|
||||
export interface Property {
|
||||
id: string;
|
||||
name: string;
|
||||
type: PropertyType;
|
||||
multiplicity: Multiplicity;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface Block {
|
||||
id: string;
|
||||
label: string;
|
||||
kind: BlockKind;
|
||||
stereotypes: string[];
|
||||
properties: Property[];
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export type AssociationKind =
|
||||
| 'association'
|
||||
| 'composition'
|
||||
| 'aggregation'
|
||||
| 'generalization'
|
||||
| 'constraintApplies';
|
||||
|
||||
export interface Association {
|
||||
id: string;
|
||||
fromBlockId: string;
|
||||
toBlockId: string;
|
||||
label: string;
|
||||
kind: AssociationKind;
|
||||
}
|
||||
|
||||
export interface Constraint {
|
||||
id: string;
|
||||
label: string;
|
||||
expression: string;
|
||||
appliesTo: string[];
|
||||
}
|
||||
|
||||
export type RequirementRelation =
|
||||
| { kind: 'satisfy'; blockId: string }
|
||||
| { kind: 'derive'; fromReqId: string };
|
||||
|
||||
export interface Requirement {
|
||||
id: string;
|
||||
tag: string;
|
||||
text: string;
|
||||
relations: RequirementRelation[];
|
||||
}
|
||||
|
||||
export interface SysMLModel {
|
||||
systemOfInterestId?: string;
|
||||
blocks: Block[];
|
||||
associations: Association[];
|
||||
constraints: Constraint[];
|
||||
requirements: Requirement[];
|
||||
}
|
||||
|
||||
// Phase-0 specific extensions
|
||||
|
||||
export interface Confidence {
|
||||
confidence: number; // 0..1
|
||||
}
|
||||
|
||||
export type GeneratedBlock = Block & Confidence;
|
||||
export type GeneratedAssociation = Association & Confidence;
|
||||
export type GeneratedConstraint = Constraint & Confidence;
|
||||
export type GeneratedRequirement = Requirement & Confidence;
|
||||
|
||||
export interface GeneratedModel {
|
||||
systemOfInterestId?: string;
|
||||
blocks: GeneratedBlock[];
|
||||
associations: GeneratedAssociation[];
|
||||
constraints: GeneratedConstraint[];
|
||||
requirements: GeneratedRequirement[];
|
||||
overallConfidence: number;
|
||||
notes?: string;
|
||||
}
|
||||
|
||||
export interface SeedPayload {
|
||||
id: string;
|
||||
title: string;
|
||||
problem: string;
|
||||
targetUser: string;
|
||||
desiredOutcome: string;
|
||||
initialHypothesis?: string;
|
||||
constraints?: string[];
|
||||
notes?: string;
|
||||
meta: {
|
||||
isHoldout: boolean;
|
||||
expectedDifficulty: 'easy' | 'medium' | 'hard' | 'failure-prone';
|
||||
testsFor: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface Finding {
|
||||
kind: 'assumption' | 'risk' | 'inconsistency';
|
||||
text: string;
|
||||
linkedElementIds: string[];
|
||||
confidence: number;
|
||||
severity?: 'low' | 'medium' | 'high';
|
||||
validationCode?: string;
|
||||
}
|
||||
|
||||
export interface ConversationTurn {
|
||||
who: 'socrates' | 'user';
|
||||
text: string;
|
||||
options?: Array<{ n: number; label: string; sub?: string }>;
|
||||
ts: string;
|
||||
}
|
||||
|
||||
export interface RubricScore {
|
||||
modelCoverage: number;
|
||||
modelAccuracy: number;
|
||||
modelParsimony: number;
|
||||
constraintCapture: number;
|
||||
assumptionDetectionQuality: number;
|
||||
riskDetectionQuality: number;
|
||||
voiceAndCharacter: number;
|
||||
comments?: string;
|
||||
scorer: 'human' | 'llm-judge';
|
||||
scorerName?: string;
|
||||
}
|
||||
|
||||
export interface RunArtifacts {
|
||||
seedId: string;
|
||||
promptVersionHash: string;
|
||||
model: GeneratedModel;
|
||||
findings: Finding[];
|
||||
conversation: ConversationTurn[];
|
||||
scores: RubricScore[];
|
||||
durationMs: number;
|
||||
tokenUsage: { input: number; output: number };
|
||||
llmModelName: string;
|
||||
}
|
||||
19
phase-0/tsconfig.json
Normal file
19
phase-0/tsconfig.json
Normal file
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": false,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist", "outputs"]
|
||||
}
|
||||
Reference in New Issue
Block a user