From f04e0dc0d4d98792d43b4d66396d8c6608850e1f Mon Sep 17 00:00:00 2001 From: dtoro Date: Tue, 7 Jul 2026 20:37:17 +0200 Subject: [PATCH] fix: PG array format for tags, entity slug prefixes, archive path handling - knowledge.go: scan tags as []string from pgx (not JSON) - seed.go: convert tags to PG array format, fix runbook applies_to_type - convert-wiki.py: fix entity slug prefixes to match inventory.yaml (host: not proxmox-host:, ws: not workstation:, service:homelab-mcp with hyphen) - convert-wiki.py: read from archive/knowledge/ since wiki was archived --- internal/httpapi/knowledge.go | 18 +- internal/knowledge/seed.go | 28 +- scripts/convert-wiki.py | 39 +- seeds/knowledge.yaml | 902 +++++++++++++++++----------------- 4 files changed, 493 insertions(+), 494 deletions(-) diff --git a/internal/httpapi/knowledge.go b/internal/httpapi/knowledge.go index 4a3a6cf..764aa5f 100644 --- a/internal/httpapi/knowledge.go +++ b/internal/httpapi/knowledge.go @@ -2,10 +2,8 @@ package httpapi import ( "context" - "encoding/json" "github.com/dtoro/oikos/internal/httpapi/gen" - "github.com/google/uuid" ) func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledgeRequestObject) (gen.SearchKnowledgeResponseObject, error) { @@ -33,17 +31,15 @@ func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledg items := []gen.KnowledgeHit{} for rows.Next() { - var slug, eType, title, source, tagJSON string + var slug, eType, title, source string + var tags []string var rank float32 var snippet *string - if err := rows.Scan(&slug, &eType, &title, &source, &tagJSON, &rank, &snippet); err != nil { + if err := rows.Scan(&slug, &eType, &title, &source, &tags, &rank, &snippet); err != nil { return nil, err } - var tags []string - json.Unmarshal([]byte(tagJSON), &tags) - hitType := gen.Document switch eType { case "investigation": @@ -52,9 +48,6 @@ func (s *Server) SearchKnowledge(ctx context.Context, request gen.SearchKnowledg hitType = gen.Runbook } - id, _ := uuid.Parse("") - _ = id // not needed for response since we use slug - items = append(items, gen.KnowledgeHit{ Slug: slug, Title: title, @@ -107,13 +100,12 @@ func (s *Server) GetEntityKnowledge(ctx context.Context, request gen.GetEntityKn items := []gen.KnowledgeHit{} for rows.Next() { - var slug, eType, title, source, tagJSON string + var slug, eType, title, source string var tags []string - if err := rows.Scan(&slug, &eType, &title, &source, &tagJSON); err != nil { + if err := rows.Scan(&slug, &eType, &title, &source, &tags); err != nil { return nil, err } - json.Unmarshal([]byte(tagJSON), &tags) hitType := gen.Document switch eType { diff --git a/internal/knowledge/seed.go b/internal/knowledge/seed.go index c436c1c..3429952 100644 --- a/internal/knowledge/seed.go +++ b/internal/knowledge/seed.go @@ -191,6 +191,9 @@ func ingestRunbook(ctx context.Context, tx pgx.Tx, m map[string]any) error { if riskClass != "" { attrs["risk_class"] = riskClass } + if entityType != "" { + attrs["applies_to_type"] = entityType + } if len(procedure) > 0 { attrs["procedure"] = procedure } @@ -204,12 +207,6 @@ func ingestRunbook(ctx context.Context, tx pgx.Tx, m map[string]any) error { } } - if entityType != "" { - if err := createEdge(ctx, tx, entitySlug, entityType, "procedure-for", nil); err != nil { - return fmt.Errorf("link runbook: %w", err) - } - } - return nil } @@ -220,7 +217,7 @@ func upsertKnowledgeEntity(ctx context.Context, tx pgx.Tx, slug, entityType, tit } hash := contentHash(content) - tagsJSON, _ := json.Marshal(tags) + tagArray := toPGArray(tags) _, err = tx.Exec(ctx, `INSERT INTO knowledge_entities (entity_id, title, content, source, tags, content_hash, created_at, updated_at) @@ -228,7 +225,7 @@ func upsertKnowledgeEntity(ctx context.Context, tx pgx.Tx, slug, entityType, tit ON CONFLICT (entity_id) DO UPDATE SET title = $2, content = $3, source = $4, tags = $5, content_hash = $6, updated_at = now()`, - id, title, content, source, string(tagsJSON), hash) + id, title, content, source, tagArray, hash) return err } @@ -273,4 +270,19 @@ func createEdge(ctx context.Context, tx pgx.Tx, sourceSlug, targetSlug, relType DO UPDATE SET attributes = EXCLUDED.attributes`, sourceID, targetID, relType, string(attrsBytes)) return err +} + +func toPGArray(tags []string) string { + if len(tags) == 0 { + return "{}" + } + out := "{" + for i, t := range tags { + if i > 0 { + out += "," + } + out += `"` + t + `"` + } + out += "}" + return out } \ No newline at end of file diff --git a/scripts/convert-wiki.py b/scripts/convert-wiki.py index 06cf561..d830876 100644 --- a/scripts/convert-wiki.py +++ b/scripts/convert-wiki.py @@ -6,9 +6,9 @@ from pathlib import Path from hashlib import sha256 REPO = Path("/Users/dtoro/Projects/oikos") -WIKI = REPO / "knowledge/wiki" -SOURCES = REPO / "knowledge/sources" -GLOSSARY = REPO / "knowledge/GLOSSARY.md" +WIKI = REPO / "archive" / "knowledge" +SOURCES = REPO / "archive" / "knowledge" +GLOSSARY = REPO / "archive" / "knowledge" / "GLOSSARY.md" # Maps wiki path components to entity slugs # Format: (path_pattern, entity_slug) @@ -35,8 +35,8 @@ PATH_TO_ENTITY = { "containers/133-seanime": "lxc:seanime", "containers/134-romm": "lxc:romm", # Hosts - "hosts/hubris": "proxmox-host:hubris", - "hosts/strong": "proxmox-host:strong", + "hosts/hubris": "host:hubris", + "hosts/strong": "host:strong", # VMs "vms/100-zimaos": "vm:zimaos", "vms/108-haos": "vm:haos", @@ -44,7 +44,7 @@ PATH_TO_ENTITY = { "infrastructure/auto-deploy": None, "infrastructure/backups": None, "infrastructure/dns": "service:dns", - "infrastructure/homelab-context": "service:homelab_mcp", + "infrastructure/homelab-context": "service:homelab-mcp", "infrastructure/ingress": "service:caddy", "infrastructure/media-permissions": "service:jellyfin", "infrastructure/mesh": None, @@ -52,7 +52,7 @@ PATH_TO_ENTITY = { "infrastructure/network": None, "infrastructure/ssh-access": None, "infrastructure/topology": None, - "infrastructure/vps-hardening": "standalone-server:netbird-vps", + "infrastructure/vps-hardening": "host:netbird-vps", } def parse_page(path): @@ -157,15 +157,8 @@ def parse_page(path): tags.append('infrastructure') # Determine slug from relative path - rel = str(path.relative_to(REPO)) - if rel.startswith('knowledge/wiki/'): - slug_rel = rel[len('knowledge/wiki/'):] - elif rel.startswith('knowledge/sources/investigations/'): - slug_rel = rel[len('knowledge/sources/'):] - else: - slug_rel = rel - - slug = slug_rel.replace('.md', '') + rel = str(path.relative_to(WIKI)) + slug = rel.replace('.md', '') # Entity mapping entity_slug = PATH_TO_ENTITY.get(slug, None) @@ -226,15 +219,15 @@ def parse_investigation(path): (r'\bartifacto\b', 'service:artifacto'), (r'\barriman\b', 'lxc:arriman'), (r'\btrmnl\b', 'service:trmnl'), - (r'\bmac-mini\b', 'workstation:mac-mini'), - (r'\bhubris\b', 'proxmox-host:hubris'), - (r'\bstrong\b', 'proxmox-host:strong'), + (r'\bmac-mini\b', 'ws:mac-mini'), + (r'\bhubris\b', 'host:hubris'), + (r'\bstrong\b', 'host:strong'), ] for pattern, slug in entity_patterns: if re.search(pattern, text, re.IGNORECASE): about_slugs.append(slug) - rel = str(path.relative_to(SOURCES)) + rel = str(path.relative_to(WIKI)) slug = rel.replace('.md', '') return { @@ -257,7 +250,9 @@ def main(): # Container pages containers_dir = WIKI / "containers" for f in sorted(containers_dir.glob("*.md")): - if 'index' in f.name or 'archive' in str(f): + if 'index' in f.name: + continue + if f.parent.name == 'archive': continue result = parse_page(f) if result and result['title']: @@ -297,7 +292,7 @@ def main(): # Investigation pages inv_dir = SOURCES / "investigations" for f in sorted(inv_dir.glob("*.md")): - if 'index' in f.name or 'archive' in str(f): + if 'index' in f.name: continue result = parse_investigation(f) if result and result['title']: diff --git a/seeds/knowledge.yaml b/seeds/knowledge.yaml index 3aec741..ea96fdb 100644 --- a/seeds/knowledge.yaml +++ b/seeds/knowledge.yaml @@ -3271,7 +3271,7 @@ documents: \ silicon hangs still leave no trace; this catches everything else.\n\n### 2026-04-21 — `cpu-epp.service` deployed\nPinned\ \ governor=`powersave`, EPP=`balance_power` at boot. Stopped the host idling at ~95 °C with everything pinned at 4.4 GHz.\ \ First fix in the [crash-loop incident](../../sources/investigations/archive/2026-04-21-hubris-crash-loop.md).\n" - entity_slug: proxmox-host:hubris + entity_slug: host:hubris tags: - host at_glance: @@ -3476,7 +3476,7 @@ documents: \ SSH pre-authorized in both directions —\nno interactive password prompt needed for the join itself. Cluster now 2\n\ nodes (`hubris`, `strong`), quorate, no QDevice. Decided to leave hostname\nas `strong` and skip a QDevice for now — both\ \ revisitable later.\n" - entity_slug: proxmox-host:strong + entity_slug: host:strong tags: - host at_glance: @@ -4586,7 +4586,7 @@ documents: \ + 11). `homelab refresh-creds` + atomic\n`client add --finalize-pubkey` grant flow live so new clients are one\nceremony\ \ instead of four manual steps. Outstanding: bootstrap mac-mini\n(macOS, exercises launchd) + ludo-mini + the remaining\ \ LXCs;\nHermes Agent integration so the agent uses inventory at chat-time.\n" - entity_slug: service:homelab_mcp + entity_slug: service:homelab-mcp tags: - infrastructure at_glance: {} @@ -5428,7 +5428,7 @@ documents: established. ' - entity_slug: standalone-server:netbird-vps + entity_slug: host:netbird-vps tags: - infrastructure at_glance: @@ -5460,450 +5460,7 @@ documents: body: nftables firewall, mesh-only SSH, fail2ban traefik jail, Plesk disabled, auto-reboot 04:00 UTC, wireguard/fail2ban invariant established. investigations: -- slug: investigations/2026-06-01-mac-mini-onboarding - title: mac-mini onboarding — post-mortem & lessons learned - date: '' - status: resolved - duration: '' - content: "# mac-mini onboarding — post-mortem & lessons learned\n\nOnboarded the `mac-mini` workstation (macOS Sequoia,\ - \ arm64) into the hubris\nhomelab context system with the `--with-hermes` profile. What follows is a\nchronological recap\ - \ of every hitch, the fix, and the systemic improvements\nneeded so the next workstation takes 5 min instead of an hour.\n\ - \n## Session log\n\n### Step 1 — clone + symlink\n- Manually cloned `git.hubris.network/dtoro/Homelab-Docs` to `/Users/dtoro/Homelab-Docs`.\n\ - - Created `/opt/homelab-context` → `/Users/dtoro/Homelab-Docs` symlink.\n- **Lesson:** bootstrap.sh was designed to do\ - \ this from scratch, but we'd\n already cloned by hand. The bootstrap's `clone exists; pulling` path handled\n it gracefully.\n\ - \n### Step 2 — hostname mismatch\n- `scutil --get LocalHostName` → `Davids-Mac-mini`\n- `hostname -s` → `Mac`\n- Inventory\ - \ file: `hosts/mac-mini.yaml`\n- **Fix:** `sudo scutil --set LocalHostName mac-mini && sudo scutil --set HostName mac-mini`\n\ - - **Lesson:** The bootstrap and `homelab whoami` use different hostname\n resolution. Bootstrap uses `scutil --get LocalHostName`\ - \ (correct on macOS),\n but the `homelab` CLI binary uses `hostname -s`. Both need to match the\n inventory key. On\ - \ a fresh macOS machine, neither does.\n\n### Step 3 — bootstrap dependencies\n- pyyaml was missing → `pip install pyyaml`\n\ - - age and sops were missing → `brew install age sops`\n- Netbird was already installed and connected ✓\n- **Lesson:**\ - \ The bootstrap preflight handles these, but only if running\n `bootstrap.sh` from the start. Since we ran it after manual\ - \ setup, some\n steps (netbird install) were correctly skipped as already-present.\n\n### Step 4 — full bootstrap with\ - \ `--with-mcp --with-hermes`\n- Ran `sudo HOMELAB_GITEA_TOKEN=... bash bootstrap.sh --with-mcp --with-hermes`\n- Age key\ - \ issued ✓\n- Launchd sync timer installed ✓\n- Goose binary installed ✓\n- Hermes CLI linked ✓\n- MCP config merged ✓\n\ - - `refresh-creds` skipped (not yet a recipient) ⚠️\n- Cosmetics: `chown: dtoro: illegal group name` at the end (benign,\ - \ macOS\n group-naming quirk)\n\n### Step 5 — finalize from hubris\n- Ran `homelab client add mac-mini --finalize-pubkey\ - \ --with-hermes` on\n hubris\n- Push failed: `[rejected] main -> main (fetch first)` — hubris clone was\n stale,\ - \ bootstrap had already pushed from mac-mini\n- **Fix:** `git pull --rebase && git push` on hubris\n- **Lesson:** bootstrap\ - \ pushes remote changes before hubris can finalize,\n creating a race. The `homelab client add --finalize-pubkey` command\ - \ should\n pull before committing/pushing.\n\n### Step 6 — sops couldn't find the age key\n- `homelab secret hello` failed\ - \ because sops looks in\n `/Users/dtoro/.ssh/id_rsa` etc. by default, not `/etc/age/key.txt`\n- The `homelab` CLI re-execs\ - \ via `sudo -E env SOPS_AGE_KEY_FILE=... sops ...`,\n but this requires passwordless sudo and the correct env var passthrough\n\ - - **Fix:**\n 1. Added NOPASSWD sudo rules\n 2. Eventually `SOPS_AGE_KEY` env with the raw key content worked directly\n\ - - **Lesson:** Document the explicit `SOPS_AGE_KEY_FILE` incantation in\n agent-enrollment troubleshooting. New clients\ - \ can't assume `homelab secret`\n works out of the gate — the sudo re-exec chain has permission pitfall.\n\n### Step\ - \ 7 — OpenRouter key was a placeholder\n- `secrets/openrouter-api-key.yaml` contained\n `api_key: PLACEHOLDER_REPLACE_WITH_REAL_OPENROUTER_KEY`\n\ - - User ran `sops` on hubris, but got the same error (age key not found on\n hubris either — `/root/.config/sops/age/keys.txt`\ - \ didn't exist)\n- **Fix:** `SOPS_AGE_KEY_FILE=/etc/age/key.txt sops ...` on hubris.\n Later: the user pasted the real\ - \ key, but the sops file showed\n `sk-or-...5c55` — the literal content was truncated with ellipsis.\n\n### Step 8 —\ - \ editor loaded the wrong data\n- Neovim on the system is configured with `clipboard+=unnamedplus`, which\n points `*`\ - \ and `+` registers to the macOS clipboard manager rather than\n X11. When editing SOPS files, this caused the **system\ - \ clipboard** to be\n pasted instead of the actual ciphertext.\n- This wasn't diagnosed during the session — the sops\ - \ file would load empty\n or show the wrong content because the editor's idea of \"paste\" was\n disconnected from what\ - \ sops expected.\n- **Fix:** Run `sops` with `EDITOR=nano` or another editor that doesn't\n hijack OS clipboards:\n \ - \ ```bash\n EDITOR=nano SOPS_AGE_KEY_FILE=/etc/age/key.txt sops secrets/openrouter-api-key.yaml\n ```\n- **Lesson:**\ - \ Add a strong warning to `hermes-agent.md` / `agent-enrollment.md`:\n macOS neovim with `clipboard+=unnamedplus` silently\ - \ breaks sops editing\n because the paste register reads from the system clipboard instead of the\n sops-managed buffer.\ - \ Use `EDITOR=nano` or `EDITOR=vim` when running sops\n interactively. Alternatively, override the clipboard option with\n\ - \ `EDITOR='nvim -c \"set clipboard=\"'`.\n- Also useful for the troubleshooting table in `agent-enrollment.md` under\ - \ a\n new row: \"sops file loads empty / wrong content on macOS\"\n\n### Step 9 — model doesn't support tool use\n- Goose\ - \ config pinned `nousresearch/hermes-4-405b` via OpenRouter\n- Error: `No endpoints found that support tool use`\n- **Fix:**\ - \ Switched to `deepseek/deepseek-v4-flash` in\n `~/.config/goose/config.yaml`\n- Also updated `operations/hermes-agent.md`\ - \ with the correct model\n- **Lesson:** The default model in `bootstrap.sh` and `hermes-agent.md` was\n never validated\ - \ on OpenRouter for tool-use capability. Need to either:\n (a) Pin a model known to work (`deepseek/deepseek-v4-flash`)\n\ - \ (b) Or have the bootstrap probe OpenRouter at install time\n\n### Step 10 — MCP homelab extension returns 404\n- Goose's\ - \ homelab extension configured as `streamable_http` at\n `https://mcp.hubris.network/mcp` returns HTTP 404\n- The actual\ - \ MCP server runs on SSE (`/sse`), not streamable_http\n- **Known follow-up** from `hermes-agent.md` — server needs migration\n\ - - **Workaround:** The agent works fine via the built-in `developer` extension\n (shell + file) and the `homelab` CLI\n\ - \n### Step 11 — git credential dance\nSeveral layers of issues with git authentication:\n\n1. **Directory permissions:**\ - \ `/etc/homelab-context` was `0700 root` — dtoro\n couldn't access the credential file, and git's `store` helper couldn't\n\ - \ create its `.lock` file.\n - **Fix 1a:** `sudo chmod 755 /etc/homelab-context`\n - **Fix 1b:** `sudo chown -R\ - \ dtoro:staff /etc/homelab-context`\n\n2. **macOS built-in osxkeychain:** Apple's git has `osxkeychain` compiled in\n\ - \ as a default credential helper. Even after configuring the `store` helper,\n git calls osxkeychain after every successful\ - \ `get`, which prompts for\n macOS keychain access.\n - `credential.helper = \"\"` does NOT disable the compiled-in\ - \ default\n - The `store` helper's `.lock` file in `/etc/homelab-context/` also failed\n silently, causing fallback\ - \ behaviour\n - **Fix:** Embed the credential in the remote URL directly:\n `https://dtoro:TOKEN@git.hubris.network/dtoro/Homelab-Docs.git`\n\ - \ - This bypasses all credential helpers entirely\n\n3. **refresh-creds:** The credential is now managed by the remote\ - \ URL.\n If the PAT is rotated, `homelab refresh-creds` won't update the URL.\n - **Workaround:** Run `git remote\ - \ set-url origin` after `refresh-creds`\n\n### Step 12 — `homelab` CLI hostname detection\n- `homelab whoami` failed with\ - \ `no hosts/Mac.yaml` because the CLI uses\n `hostname -s` (returning `Mac`) while the bootstrap uses\n `scutil --get\ - \ LocalHostName` (which we fixed to `mac-mini`)\n- **Fix:** Set both `LocalHostName` and `HostName` via `scutil`\n- **Lesson:**\ - \ The homelab CLI on macOS should prefer `scutil --get\n LocalHostName` like the bootstrap does, or at least try both\ - \ and pick the\n one that matches an inventory key. This is a CLI bug.\n\n## Improvement backlog\n\n### High priority\n\ - \n| # | Issue | Fix |\n|---|-------|-----|\n| 1 | Homelab CLI uses `hostname -s` on macOS; should use `scutil --get LocalHostName`\ - \ to match bootstrap | Patch `bin/homelab` to try `scutil` first, fall back to `hostname -s` |\n| 2 | `homelab client\ - \ add --finalize-pubkey` on hubris fails to push when remote is ahead (bootstrap pushes first) | `pull --rebase` before\ - \ committing in `cmd_client_add` |\n| 3 | Default Goose model (`nousresearch/hermes-4-405b`) doesn't support tool use\ - \ on OpenRouter | Update `bootstrap.sh` default and `hermes-agent.md` to `deepseek/deepseek-v4-flash` |\n| 4 | MCP server\ - \ needs streamable_http migration | Follow-up #1 from `hermes-agent.md` — FastMCP `transport=\"sse\"` → `transport=\"\ - streamable_http\"` |\n| 5 | `homelab refresh-creds` should also update embedded remote URLs | In `cmd_refresh_creds`,\ - \ after writing the credential file, update any remote URL that has an embedded credential |\n\n### Medium priority\n\n\ - | # | Issue | Fix |\n|---|-------|-----|\n| 6 | macOS neovim with `clipboard+=unnamedplus` breaks sops editing | Add troubleshooting\ - \ row to `agent-enrollment.md`; recommend `EDITOR=nano` |\n| 7 | `/etc/homelab-context` directory with 0700 root permissions\ - \ blocks non-root git | Change to 755 in `bootstrap.sh` on all OSes (or at least macOS) |\n| 8 | Bootstrap `chown` at\ - \ end gives `illegal group name` on macOS | Fix group detection in bootstrap.sh for macOS |\n| 9 | `homelab secret` re-exec\ - \ via sudo chain is fragile — needs passwordless sudo or TTY | Consider `SOPS_AGE_KEY` env-var fallback path in the CLI\ - \ itself |\n\n### Low priority\n\n| # | Issue | Fix |\n|---|-------|-----|\n| 10 | Onboarding this machine revealed no\ - \ `HERMES.md` file in the repo | Create the persona file for the Goose hints symlink |\n| 11 | Git credential `store`\ - \ helper lock mechanism is OS-sensitive and fragile | Document embedded-URL pattern as the preferred approach for workstations\ - \ |\n| 12 | AGENTS.md references `/opt/homelab-context/hosts/.yaml` but the canonical path is now a symlink\ - \ | Verify the path resolution edge cases |\n\n## Changelog\n\n### 2026-06-01 — initial post-mortem\nCaptured the full\ - \ onboarding of mac-mini into the hubris homelab context\nsystem, including the hostname mismatch, sudo/sops credential\ - \ chain, model\nselection, git credential gymnastics, and editor clipboard interference.\n\n### 2026-06-02 — MCP server\ - \ migrated to streamable-http\nItem #4 resolved. FastMCP server now uses `transport=\"streamable-http\"` (hyphen,\nnot\ - \ underscore), serving at `https://mcp.hubris.network/mcp`. All doc references\nand client configs (Goose, bootstrap.sh,\ - \ agent-enrollment.md) updated. Service\nre-deployed on apps LXC (105).\n" - about_slugs: - - workstation:mac-mini - - proxmox-host:hubris - - proxmox-host:strong - tags: - - investigation -- slug: investigations/2026-06-03-moonlight-sunshine-wifi-jitter - title: 2026-06-03 — Moonlight/Sunshine game streaming unstable over WiFi - date: '2026-06-03' - status: resolved - duration: '' - content: '# 2026-06-03 — Moonlight/Sunshine game streaming unstable over WiFi - - - ## Summary - - - [`ludo-mini`](../../../hosts/strong.yaml) runs Sunshine as the game-streaming server; [`mac-mini`](../../../hosts/mac-mini.yaml) - runs Moonlight as the client. Despite both machines being on the same physical subnet (192.168.178.0/24), streaming was - unstable — stuttering, dropouts, and high latency. Root cause: **mac-mini is connected only via WiFi**, while ludo-mini - is wired Ethernet (2.5 Gbps). WiFi throughput shows 1-second UDP dropouts and high jitter (28 ms stddev), which breaks - real-time video streaming. - - - ## Timeline - - - ### Pre-existing state - - - ludo-mini: wired Ethernet (eno1, 2.5 Gbps), 192.168.178.181/24 - - - mac-mini: WiFi (en1, 802.11ac, 234 Mbps Tx rate, 1 stream, -60 dBm RSSI), 192.168.178.99/24 - - - Both on the same subnet via a consumer router at 192.168.178.1 - - - Sunshine configured: bitrate=80 Mbps, encoder=amf (AMD GPU), fec_percentage=5, hevc_mode=2 - - - Moonlight configured: bitrate=57 Mbps, fps=60, localaddr=192.168.178.181 (correct) - - - ### 2026-06-03 — investigation - - **Network tests (iperf3 between mac-mini ↔ ludo-mini):** - - - TCP throughput: 49–86 Mbps, average ~65 Mbps — highly variable - - - UDP throughput: 1-second complete dropout during a 5-second test; 0% loss otherwise - - - Ping: 4.7 ms avg, but **28 ms stddev**, max **138 ms** — WiFi-level jitter - - - Netbird P2P connection also showed 7 ms latency over the tunnel (unnecessary given same-subnet direct connection) - - - **Additional findings:** - - - mac-mini has a built-in Ethernet port (en0) but it is inactive — no cable connected - - - mac-mini also has Thunderbolt Ethernet adapters (en5, en6, en7) all inactive - - - ludo-mini''s en0 shows `speed 2500` (2.5 GbE) - - - Netbird routes `192.168.8.0/24` via `utun100`, but 192.168.178.0/24 traffic stays on `en1` - - - ## Root cause - - - mac-mini is on WiFi. WiFi introduces: - - 1. **Throughput variance** — 49–86 Mbps TCP fluctuation - - 2. **Dropouts** — 1-second UDP blackouts from radio retransmissions - - 3. **Jitter** — 28 ms stddev with spikes to 138 ms - - - Moonlight/Sunshine streams real-time video over UDP. Any jitter spike or dropout causes visual stutter or frame drops. - The 57 Mbps Moonlight bitrate was too aggressive for the WiFi link''s consistency, even though the average throughput - is ~65 Mbps. - - - ## Mitigations applied - - - ### mac-mini — Moonlight (via `defaults write`) - - - | Setting | Before | After | Rationale | - - ||---------|--------|-------|-----------| - - || bitrate | 57 Mbps | 25 Mbps | Reduce to a level WiFi can sustain consistently | - - || framepacing | off (0) | on (1) | Smooths frame timing over variable latency | - - || fps | 30 | 60 | Restored to game-appropriate value | - - - ### mac-mini — Moonlight (current, wired — 2026-06-04) - - - | Setting | WiFi value | Wired value | Rationale | - - ||---------|-----------|-------------|-----------| - - || bitrate | 60 Mbps | 80 Mbps | Wired 1 GbE can sustain comfortably | - - || framepacing | on (1) | off (0) | Wired latency is consistent, no smoothing needed | - - || fps | 60 | 60 | Unchanged | - - - ### ludo-mini — Sunshine (via SSH) - - - | Setting | WiFi value | Wired value | Rationale | - - ||---------|-----------|-------------|-----------| - - || bitrate | 40 Mbps | 80 Mbps | Wired can handle full bandwidth | - - || fec_percentage | 2 | 5 | Restored to default — wired has no dropout concern | - - || packetsize | 1024 | 1316 | Restored to default (Ethernet MTU) | - - - Sunshine restarted after changes; confirmed active. - - - ## Resolution - - - ### 2026-06-04 — mac-mini wired to LAN - - - mac-mini plugged into Ethernet (en0, 192.168.178.182). This definitively resolves the WiFi jitter that caused streaming - instability. All WiFi-era conservative settings reverted to appropriate wired values on both ends. - - - ## Open questions - - - ~~1. **Ethernet wiring** — mac-mini has an active en0 port. Plugging it into the same switch/router as ludo-mini would - eliminate WiFi jitter entirely. This is the definitive fix.~~ **RESOLVED** — mac-mini now wired via en0. - - 2. **WiFi alternatives** — If wiring is impractical, a WiFi 6 (802.11ax) USB adapter or Thunderbolt-to-Ethernet adapter - would help, but wired Ethernet is the only reliable solution for game streaming. - - 3. **Netbird route** — `192.168.8.0/24` is routed over Netbird `utun100`. If Moonlight ever falls back to that subnet - (ludo-mini''s 192.168.8.133 LAN IP), traffic tunnels unnecessarily. Moonlight correctly uses 192.168.178.181, so this - is not currently an issue. - - - ## Changelog - - - ### 2026-06-04 — mac-mini wired to LAN, configs reverted from WiFi mitigations - - - mac-mini connected via en0 (wired Ethernet, 192.168.178.182), WiFi mitigations no longer needed - - - Moonlight: bitrate 60→80 Mbps, framepacing disabled - - - Sunshine: bitrate 40→80 Mbps, fec_percentage 2→5, packetsize restored to default (1316) - - - First root cause in investigation (Open questions #1) resolved' - about_slugs: - - workstation:mac-mini - - proxmox-host:strong - tags: - - investigation -- slug: investigations/2026-06-06-authentik-session-lifetime - title: 2026-06-06 — Frequent Authentik login prompts fixed (session duration) - date: '2026-06-06' - status: resolved - duration: '' - content: "# 2026-06-06 — Frequent Authentik login prompts fixed (session duration)\n\n## Summary\n\nUser needed to re-authenticate\ - \ to Authentik several times per day. Root cause was the Django session being configured as a **session cookie** (cleared\ - \ on browser close) with a short 24-hour lifetime. Fixed by extending both the session cookie lifetime and the user_login\ - \ stage session duration to 30 days.\n\n## Timeline\n\n- **2026-06-06 ~23:00** — User reports \"having to login to authentik\ - \ several times per day\"\n- **~23:10** — Investigation begins: check Authentik OAuth2 provider settings, proxy session\ - \ table, Django session config\n- **~23:20** — Found `SESSION_EXPIRE_AT_BROWSER_CLOSE=True` — the `authentik_session`\ - \ cookie has no Max-Age, cleared on browser close\n- **~23:25** — Found the `user_login` stage has `session_duration=seconds=0`,\ - \ which calls `session.set_expiry(0)` → `_session_expire_at_browser_close=True`\n- **~23:30** — Applied DB fix: updated\ - \ `authentik_stages_user_login_userloginstage` → `session_duration='days=30'`\n- **~23:35** — Added `AUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE=days=30`\ - \ to `/opt/authentik.env` (maps to `sessions.unauthenticated_age` in Authentik config, setting `SESSION_COOKIE_AGE` to\ - \ 30 days)\n- **~23:40** — Recreated container with `docker compose up -d` (restart doesn't re-read env file)\n- **~23:45**\ - \ — Verified both changes took effect\n\n## Root cause\n\nTwo independent but compounding issues:\n\n1. **`SESSION_EXPIRE_AT_BROWSER_CLOSE=True`**\ - \ — hardcoded in `/authentik/root/settings.py`. Makes the `authentik_session` cookie a session cookie (no `Max-Age`),\ - \ cleared when the browser closes. Cannot be changed via env vars or config files (hardcoded in Python source).\n\n2.\ - \ **`user_login` stage `session_duration=seconds=0`** — the default-authentication-login stage called `session.set_expiry(0)`,\ - \ which in Django sets `_session_expire_at_browser_close=True` on the session object, overriding any session cookie lifetime\ - \ set via `SESSION_COOKIE_AGE`.\n\n3. **`sessions.unauthenticated_age=days=1`** (default) — `SESSION_COOKIE_AGE` was 86,400\ - \ seconds (24 hours), so even with the browser left open, the server-side session data expired after 24 hours with no\ - \ way to extend it (`SESSION_SAVE_EVERY_REQUEST=False`).\n\n### Why forward-auth worked but OAuth2 didn't\n\n- **Forward-auth\ - \ (Caddy-gated services)** — The `authentik_proxy_*` cookie is a *persistent* cookie with 24-hour validity, set by the\ - \ Authentik proxy outpost. It survives browser restart. Caddy's forward-auth validates this cookie directly with the outpost\ - \ API — no Django session needed.\n- **OAuth2 (Gitea, Jellyfin, etc.)** — These services redirect to `auth.hubris.network/application/o/authorize/`,\ - \ which checks the Django session (`authentik_session` cookie). If the browser was closed, this cookie is gone → user\ - \ is redirected to the login form.\n\n## Changes applied\n\n### 1. Database — `authentik_stages_user_login_userloginstage`\n\ - \n```sql\nUPDATE authentik_stages_user_login_userloginstage\nSET session_duration = 'days=30'\nWHERE stage_ptr_id = (\n\ - \ SELECT stage_uuid FROM authentik_flows_stage \n WHERE name = 'default-authentication-login'\n);\n```\n\nThis causes\ - \ the login stage to call `session.set_expiry(timedelta(days=30))`, which:\n- Sets `_session_expire_at_browser_close=False`\ - \ for that session\n- Sets the session cookie `Max-Age` to 30 days (persistent cookie)\n- Sets the server-side session\ - \ expiry to 30 days\n\n### 2. Environment — `AUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE`\n\nAdded to `/opt/authentik.env`:\n\ - \n```\nAUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE=days=30\n```\n\nThis maps to config key `sessions.unauthenticated_age`,\ - \ which Authentik's `settings.py` reads via:\n\n```python\nSESSION_COOKIE_AGE = timedelta_from_string(\n CONFIG.get(\"\ - sessions.unauthenticated_age\", \"days=1\")\n).total_seconds()\n```\n\nResult: `SESSION_COOKIE_AGE` = 2,592,000 seconds\ - \ (30 days). This is a fallback for sessions that don't go through the `user_login` stage or when the stage's explicit\ - \ `set_expiry` doesn't apply.\n\n## Verification\n\n```python\n# Inside authentik-server container:\nimport os\nos.environ.setdefault(\"\ - DJANGO_SETTINGS_MODULE\", \"authentik.root.settings\")\nimport django\ndjango.setup()\nfrom django.conf import settings\n\ - print(\"SESSION_COOKIE_AGE:\", settings.SESSION_COOKIE_AGE) # → 2592000.0\nprint(\"SESSION_EXPIRE_AT_BROWSER_CLOSE:\"\ - , settings.SESSION_EXPIRE_AT_BROWSER_CLOSE) # → True\n\nfrom authentik.stages.user_login.models import UserLoginStage\n\ - stage = UserLoginStage.objects.filter(name=\"default-authentication-login\").first()\nprint(\"session_duration:\", stage.session_duration)\ - \ # → \"days=30\"\n```\n\n## Open questions\n\n- `SESSION_COOKIE_SECURE=False` — Should be `True` since Authentik is\ - \ HTTPS-only, but the custom `SessionMiddleware.is_secure()` method dynamically sets the cookie's `secure` flag based\ - \ on the request, so it's fine.\n- 30 days is generous. Could be reduced to 7-14 days if desired. Change the DB value\ - \ and env var accordingly.\n\n## Related\n\n- [Container 106 — auth-outpost](../../wiki/containers/106-auth-outpost.md)\n\ - - [Authentik VPS migration](archive/2026-05-31-authentik-vps-migration.md)\n- [Ingress (VPS Traefik)](../../wiki/infrastructure/ingress.md)\n\ - - `.hermes/plans/2026-06-06_232200-authentik-frequent-login-fix.md` — original plan\n\n## Changelog\n\n### 2026-06-06\ - \ — created\nDocumented the session lifetime investigation, root cause, and applied fixes." - about_slugs: - - service:caddy - - service:authentik - - service:gitea - - service:jellyfin - - proxmox-host:hubris - tags: - - investigation -- slug: investigations/2026-06-06-caddyfile-truncation - title: 'Investigation: Caddyfile truncation — all LAN services down (2026-06-06)' - date: '2026-06-06' - status: resolved - duration: ~10 hours (from last known good state ~12:39 UTC to restoration ~22:40 UTC) - content: '# Investigation: Caddyfile truncation — all LAN services down (2026-06-06) - - - **Date:** 2026-06-06 - - **Status:** resolved - - **Duration:** ~10 hours (from last known good state ~12:39 UTC to restoration ~22:40 UTC) - - - ## Symptom - - - All `*.hubris.network` URLs except `photos.hubris.network` and `auth.hubris.network` returned `tlsv1 alert internal error` - or TCP timeouts from LAN/mesh clients. `dig @192.168.8.2` and `dig @100.122.255.254` both resolved to `192.168.8.175` - correctly — DNS was fine. The issue was at the Caddy level. - - - ## Root cause - - - The Caddyfile on LXC 121 was manually edited directly on the filesystem (not via the `dtoro/caddy-conf` git repo), reducing - it from 260 lines/30+ site blocks to 43 lines with only 3 photo-related site blocks: `photos.hubris.network`, `prism.hubris.network`, - and `photos2.hubris.network`. - - - ### Timeline - - - | Time (UTC+2) | Event | - - |---|---| - - | Jun 04 23:43 | Last successful git-push deploy — full Caddyfile (260 lines) | - - | Jun 06 ~12:00 | Caddyfile manually edited locally, truncating to 3 sites | - - | Jun 06 12:39 | Deploy webhook triggered → `git pull --ff-only` failed: "Your local changes would be overwritten" | - - | Jun 06 14:13 | Deploy webhook triggered again → `deploy ok` (the truncated file was committed or merged somehow) | - - | Jun 06 22:34 | Investigation began | - - | Jun 06 22:43 | Caddyfile restored from `origin/master`, `systemctl reload caddy` | - - - ### Evidence - - - - `git diff HEAD -- Caddyfile` on LXC 121: `+3 / -159` lines - - - Git reflog: HEAD at `32575ce` (fix: sab port 8081→8082), working tree diverged - - - Backup file `Caddyfile.bak.1780263919`: 225 lines, full original config - - - `git stash list` shows one auto-stash entry - - - `origin/master` at `1b977aa`: 260 lines, all site blocks present - - - ### Secondary root cause found during investigation - - - **elementsynapse (LXC 118)** had `iface eth0 inet dhcp` internally despite `pct set 118 --net0 ... ip=192.168.8.239/24`. - On DHCP lease renewal, dhclient grabbed `.244` from Technitium''s pool. Caddy''s `reverse_proxy 192.168.8.239:8008` was - hitting a dead IP. - - - This is the same class of drift as the June 5th incidents (paperless, HAOS, apps, mule-images). Elementsynapse was missed - during the 2026-06-02 static-IP migration. - - - ## Fix applied - - - 1. **Caddyfile** → `git checkout --force origin/master -- Caddyfile` + `systemctl reload caddy` - - 2. **elementsynapse** → replaced `iface eth0 inet dhcp` with static, killed dhclient, verified connectivity - - - ## Permanent safeguards (all deployed) - - - | Safeguard | Location | What it does | - - |---|---|---| - - | Site-count guard | `/etc/caddy/scripts/deploy.sh` | Refuses reload if <20 `hubris.network` site blocks | - - | Dirty-tree auto-stash | `/etc/caddy/scripts/deploy.sh` | Stashes local edits before git pull | - - | Auto-backup | `/etc/caddy/scripts/deploy.sh` | Saves Caddyfile.bak. before any change, keeps 5 | - - | Caddy backend health | `/etc/cron.d/caddy-backend-health` on hubris | Runs `check-caddy-backends.sh` every 10 min | - - | DNS sync | `/etc/cron.d/dns-sync` on LXC 107 | Runs `dns-sync.py` every 10 min (was missing since 2026-06-04) | - - - ## Related - - - - DHCP drift investigation (previous incident) — not filed as its own investigation; see the [DNS sync fix](../../../.hermes/plans/2026-06-05_170000-prevent-dhcp-ip-drift.md) - - - [Caddy (121)](../../wiki/containers/121-caddy.md) - - - [elementsynapse (118)](../../wiki/containers/118-elementsynapse.md) - - - [dns-sync script](../../../scripts/dns-sync.py) - - - [check-caddy-backends script](../../../scripts/check-caddy-backends.sh)' - about_slugs: - - service:caddy - - service:dns - - service:paperless - - proxmox-host:hubris - tags: - - investigation -- slug: investigations/archive/2026-04-21-hubris-crash-loop +- slug: investigations/2026-04-21-hubris-crash-loop title: 2026-04-21 — Hubris crash loop (thermal + USB drive) date: '2026-04-21' status: resolved @@ -6106,10 +5663,10 @@ investigations: ' about_slugs: - - proxmox-host:hubris + - host:hubris tags: - investigation -- slug: investigations/archive/2026-05-31-authentik-vps-migration +- slug: investigations/2026-05-31-authentik-vps-migration title: 2026-05-31 — Authentik migrated from LXC 124 to the VPS date: '2026-05-31' status: resolved @@ -6361,7 +5918,450 @@ investigations: - service:dns - service:paperless - service:artifacto - - proxmox-host:hubris + - host:hubris + tags: + - investigation +- slug: investigations/2026-06-01-mac-mini-onboarding + title: mac-mini onboarding — post-mortem & lessons learned + date: '' + status: resolved + duration: '' + content: "# mac-mini onboarding — post-mortem & lessons learned\n\nOnboarded the `mac-mini` workstation (macOS Sequoia,\ + \ arm64) into the hubris\nhomelab context system with the `--with-hermes` profile. What follows is a\nchronological recap\ + \ of every hitch, the fix, and the systemic improvements\nneeded so the next workstation takes 5 min instead of an hour.\n\ + \n## Session log\n\n### Step 1 — clone + symlink\n- Manually cloned `git.hubris.network/dtoro/Homelab-Docs` to `/Users/dtoro/Homelab-Docs`.\n\ + - Created `/opt/homelab-context` → `/Users/dtoro/Homelab-Docs` symlink.\n- **Lesson:** bootstrap.sh was designed to do\ + \ this from scratch, but we'd\n already cloned by hand. The bootstrap's `clone exists; pulling` path handled\n it gracefully.\n\ + \n### Step 2 — hostname mismatch\n- `scutil --get LocalHostName` → `Davids-Mac-mini`\n- `hostname -s` → `Mac`\n- Inventory\ + \ file: `hosts/mac-mini.yaml`\n- **Fix:** `sudo scutil --set LocalHostName mac-mini && sudo scutil --set HostName mac-mini`\n\ + - **Lesson:** The bootstrap and `homelab whoami` use different hostname\n resolution. Bootstrap uses `scutil --get LocalHostName`\ + \ (correct on macOS),\n but the `homelab` CLI binary uses `hostname -s`. Both need to match the\n inventory key. On\ + \ a fresh macOS machine, neither does.\n\n### Step 3 — bootstrap dependencies\n- pyyaml was missing → `pip install pyyaml`\n\ + - age and sops were missing → `brew install age sops`\n- Netbird was already installed and connected ✓\n- **Lesson:**\ + \ The bootstrap preflight handles these, but only if running\n `bootstrap.sh` from the start. Since we ran it after manual\ + \ setup, some\n steps (netbird install) were correctly skipped as already-present.\n\n### Step 4 — full bootstrap with\ + \ `--with-mcp --with-hermes`\n- Ran `sudo HOMELAB_GITEA_TOKEN=... bash bootstrap.sh --with-mcp --with-hermes`\n- Age key\ + \ issued ✓\n- Launchd sync timer installed ✓\n- Goose binary installed ✓\n- Hermes CLI linked ✓\n- MCP config merged ✓\n\ + - `refresh-creds` skipped (not yet a recipient) ⚠️\n- Cosmetics: `chown: dtoro: illegal group name` at the end (benign,\ + \ macOS\n group-naming quirk)\n\n### Step 5 — finalize from hubris\n- Ran `homelab client add mac-mini --finalize-pubkey\ + \ --with-hermes` on\n hubris\n- Push failed: `[rejected] main -> main (fetch first)` — hubris clone was\n stale,\ + \ bootstrap had already pushed from mac-mini\n- **Fix:** `git pull --rebase && git push` on hubris\n- **Lesson:** bootstrap\ + \ pushes remote changes before hubris can finalize,\n creating a race. The `homelab client add --finalize-pubkey` command\ + \ should\n pull before committing/pushing.\n\n### Step 6 — sops couldn't find the age key\n- `homelab secret hello` failed\ + \ because sops looks in\n `/Users/dtoro/.ssh/id_rsa` etc. by default, not `/etc/age/key.txt`\n- The `homelab` CLI re-execs\ + \ via `sudo -E env SOPS_AGE_KEY_FILE=... sops ...`,\n but this requires passwordless sudo and the correct env var passthrough\n\ + - **Fix:**\n 1. Added NOPASSWD sudo rules\n 2. Eventually `SOPS_AGE_KEY` env with the raw key content worked directly\n\ + - **Lesson:** Document the explicit `SOPS_AGE_KEY_FILE` incantation in\n agent-enrollment troubleshooting. New clients\ + \ can't assume `homelab secret`\n works out of the gate — the sudo re-exec chain has permission pitfall.\n\n### Step\ + \ 7 — OpenRouter key was a placeholder\n- `secrets/openrouter-api-key.yaml` contained\n `api_key: PLACEHOLDER_REPLACE_WITH_REAL_OPENROUTER_KEY`\n\ + - User ran `sops` on hubris, but got the same error (age key not found on\n hubris either — `/root/.config/sops/age/keys.txt`\ + \ didn't exist)\n- **Fix:** `SOPS_AGE_KEY_FILE=/etc/age/key.txt sops ...` on hubris.\n Later: the user pasted the real\ + \ key, but the sops file showed\n `sk-or-...5c55` — the literal content was truncated with ellipsis.\n\n### Step 8 —\ + \ editor loaded the wrong data\n- Neovim on the system is configured with `clipboard+=unnamedplus`, which\n points `*`\ + \ and `+` registers to the macOS clipboard manager rather than\n X11. When editing SOPS files, this caused the **system\ + \ clipboard** to be\n pasted instead of the actual ciphertext.\n- This wasn't diagnosed during the session — the sops\ + \ file would load empty\n or show the wrong content because the editor's idea of \"paste\" was\n disconnected from what\ + \ sops expected.\n- **Fix:** Run `sops` with `EDITOR=nano` or another editor that doesn't\n hijack OS clipboards:\n \ + \ ```bash\n EDITOR=nano SOPS_AGE_KEY_FILE=/etc/age/key.txt sops secrets/openrouter-api-key.yaml\n ```\n- **Lesson:**\ + \ Add a strong warning to `hermes-agent.md` / `agent-enrollment.md`:\n macOS neovim with `clipboard+=unnamedplus` silently\ + \ breaks sops editing\n because the paste register reads from the system clipboard instead of the\n sops-managed buffer.\ + \ Use `EDITOR=nano` or `EDITOR=vim` when running sops\n interactively. Alternatively, override the clipboard option with\n\ + \ `EDITOR='nvim -c \"set clipboard=\"'`.\n- Also useful for the troubleshooting table in `agent-enrollment.md` under\ + \ a\n new row: \"sops file loads empty / wrong content on macOS\"\n\n### Step 9 — model doesn't support tool use\n- Goose\ + \ config pinned `nousresearch/hermes-4-405b` via OpenRouter\n- Error: `No endpoints found that support tool use`\n- **Fix:**\ + \ Switched to `deepseek/deepseek-v4-flash` in\n `~/.config/goose/config.yaml`\n- Also updated `operations/hermes-agent.md`\ + \ with the correct model\n- **Lesson:** The default model in `bootstrap.sh` and `hermes-agent.md` was\n never validated\ + \ on OpenRouter for tool-use capability. Need to either:\n (a) Pin a model known to work (`deepseek/deepseek-v4-flash`)\n\ + \ (b) Or have the bootstrap probe OpenRouter at install time\n\n### Step 10 — MCP homelab extension returns 404\n- Goose's\ + \ homelab extension configured as `streamable_http` at\n `https://mcp.hubris.network/mcp` returns HTTP 404\n- The actual\ + \ MCP server runs on SSE (`/sse`), not streamable_http\n- **Known follow-up** from `hermes-agent.md` — server needs migration\n\ + - **Workaround:** The agent works fine via the built-in `developer` extension\n (shell + file) and the `homelab` CLI\n\ + \n### Step 11 — git credential dance\nSeveral layers of issues with git authentication:\n\n1. **Directory permissions:**\ + \ `/etc/homelab-context` was `0700 root` — dtoro\n couldn't access the credential file, and git's `store` helper couldn't\n\ + \ create its `.lock` file.\n - **Fix 1a:** `sudo chmod 755 /etc/homelab-context`\n - **Fix 1b:** `sudo chown -R\ + \ dtoro:staff /etc/homelab-context`\n\n2. **macOS built-in osxkeychain:** Apple's git has `osxkeychain` compiled in\n\ + \ as a default credential helper. Even after configuring the `store` helper,\n git calls osxkeychain after every successful\ + \ `get`, which prompts for\n macOS keychain access.\n - `credential.helper = \"\"` does NOT disable the compiled-in\ + \ default\n - The `store` helper's `.lock` file in `/etc/homelab-context/` also failed\n silently, causing fallback\ + \ behaviour\n - **Fix:** Embed the credential in the remote URL directly:\n `https://dtoro:TOKEN@git.hubris.network/dtoro/Homelab-Docs.git`\n\ + \ - This bypasses all credential helpers entirely\n\n3. **refresh-creds:** The credential is now managed by the remote\ + \ URL.\n If the PAT is rotated, `homelab refresh-creds` won't update the URL.\n - **Workaround:** Run `git remote\ + \ set-url origin` after `refresh-creds`\n\n### Step 12 — `homelab` CLI hostname detection\n- `homelab whoami` failed with\ + \ `no hosts/Mac.yaml` because the CLI uses\n `hostname -s` (returning `Mac`) while the bootstrap uses\n `scutil --get\ + \ LocalHostName` (which we fixed to `mac-mini`)\n- **Fix:** Set both `LocalHostName` and `HostName` via `scutil`\n- **Lesson:**\ + \ The homelab CLI on macOS should prefer `scutil --get\n LocalHostName` like the bootstrap does, or at least try both\ + \ and pick the\n one that matches an inventory key. This is a CLI bug.\n\n## Improvement backlog\n\n### High priority\n\ + \n| # | Issue | Fix |\n|---|-------|-----|\n| 1 | Homelab CLI uses `hostname -s` on macOS; should use `scutil --get LocalHostName`\ + \ to match bootstrap | Patch `bin/homelab` to try `scutil` first, fall back to `hostname -s` |\n| 2 | `homelab client\ + \ add --finalize-pubkey` on hubris fails to push when remote is ahead (bootstrap pushes first) | `pull --rebase` before\ + \ committing in `cmd_client_add` |\n| 3 | Default Goose model (`nousresearch/hermes-4-405b`) doesn't support tool use\ + \ on OpenRouter | Update `bootstrap.sh` default and `hermes-agent.md` to `deepseek/deepseek-v4-flash` |\n| 4 | MCP server\ + \ needs streamable_http migration | Follow-up #1 from `hermes-agent.md` — FastMCP `transport=\"sse\"` → `transport=\"\ + streamable_http\"` |\n| 5 | `homelab refresh-creds` should also update embedded remote URLs | In `cmd_refresh_creds`,\ + \ after writing the credential file, update any remote URL that has an embedded credential |\n\n### Medium priority\n\n\ + | # | Issue | Fix |\n|---|-------|-----|\n| 6 | macOS neovim with `clipboard+=unnamedplus` breaks sops editing | Add troubleshooting\ + \ row to `agent-enrollment.md`; recommend `EDITOR=nano` |\n| 7 | `/etc/homelab-context` directory with 0700 root permissions\ + \ blocks non-root git | Change to 755 in `bootstrap.sh` on all OSes (or at least macOS) |\n| 8 | Bootstrap `chown` at\ + \ end gives `illegal group name` on macOS | Fix group detection in bootstrap.sh for macOS |\n| 9 | `homelab secret` re-exec\ + \ via sudo chain is fragile — needs passwordless sudo or TTY | Consider `SOPS_AGE_KEY` env-var fallback path in the CLI\ + \ itself |\n\n### Low priority\n\n| # | Issue | Fix |\n|---|-------|-----|\n| 10 | Onboarding this machine revealed no\ + \ `HERMES.md` file in the repo | Create the persona file for the Goose hints symlink |\n| 11 | Git credential `store`\ + \ helper lock mechanism is OS-sensitive and fragile | Document embedded-URL pattern as the preferred approach for workstations\ + \ |\n| 12 | AGENTS.md references `/opt/homelab-context/hosts/.yaml` but the canonical path is now a symlink\ + \ | Verify the path resolution edge cases |\n\n## Changelog\n\n### 2026-06-01 — initial post-mortem\nCaptured the full\ + \ onboarding of mac-mini into the hubris homelab context\nsystem, including the hostname mismatch, sudo/sops credential\ + \ chain, model\nselection, git credential gymnastics, and editor clipboard interference.\n\n### 2026-06-02 — MCP server\ + \ migrated to streamable-http\nItem #4 resolved. FastMCP server now uses `transport=\"streamable-http\"` (hyphen,\nnot\ + \ underscore), serving at `https://mcp.hubris.network/mcp`. All doc references\nand client configs (Goose, bootstrap.sh,\ + \ agent-enrollment.md) updated. Service\nre-deployed on apps LXC (105).\n" + about_slugs: + - ws:mac-mini + - host:hubris + - host:strong + tags: + - investigation +- slug: investigations/2026-06-03-moonlight-sunshine-wifi-jitter + title: 2026-06-03 — Moonlight/Sunshine game streaming unstable over WiFi + date: '2026-06-03' + status: resolved + duration: '' + content: '# 2026-06-03 — Moonlight/Sunshine game streaming unstable over WiFi + + + ## Summary + + + [`ludo-mini`](../../../hosts/strong.yaml) runs Sunshine as the game-streaming server; [`mac-mini`](../../../hosts/mac-mini.yaml) + runs Moonlight as the client. Despite both machines being on the same physical subnet (192.168.178.0/24), streaming was + unstable — stuttering, dropouts, and high latency. Root cause: **mac-mini is connected only via WiFi**, while ludo-mini + is wired Ethernet (2.5 Gbps). WiFi throughput shows 1-second UDP dropouts and high jitter (28 ms stddev), which breaks + real-time video streaming. + + + ## Timeline + + + ### Pre-existing state + + - ludo-mini: wired Ethernet (eno1, 2.5 Gbps), 192.168.178.181/24 + + - mac-mini: WiFi (en1, 802.11ac, 234 Mbps Tx rate, 1 stream, -60 dBm RSSI), 192.168.178.99/24 + + - Both on the same subnet via a consumer router at 192.168.178.1 + + - Sunshine configured: bitrate=80 Mbps, encoder=amf (AMD GPU), fec_percentage=5, hevc_mode=2 + + - Moonlight configured: bitrate=57 Mbps, fps=60, localaddr=192.168.178.181 (correct) + + + ### 2026-06-03 — investigation + + **Network tests (iperf3 between mac-mini ↔ ludo-mini):** + + - TCP throughput: 49–86 Mbps, average ~65 Mbps — highly variable + + - UDP throughput: 1-second complete dropout during a 5-second test; 0% loss otherwise + + - Ping: 4.7 ms avg, but **28 ms stddev**, max **138 ms** — WiFi-level jitter + + - Netbird P2P connection also showed 7 ms latency over the tunnel (unnecessary given same-subnet direct connection) + + + **Additional findings:** + + - mac-mini has a built-in Ethernet port (en0) but it is inactive — no cable connected + + - mac-mini also has Thunderbolt Ethernet adapters (en5, en6, en7) all inactive + + - ludo-mini''s en0 shows `speed 2500` (2.5 GbE) + + - Netbird routes `192.168.8.0/24` via `utun100`, but 192.168.178.0/24 traffic stays on `en1` + + + ## Root cause + + + mac-mini is on WiFi. WiFi introduces: + + 1. **Throughput variance** — 49–86 Mbps TCP fluctuation + + 2. **Dropouts** — 1-second UDP blackouts from radio retransmissions + + 3. **Jitter** — 28 ms stddev with spikes to 138 ms + + + Moonlight/Sunshine streams real-time video over UDP. Any jitter spike or dropout causes visual stutter or frame drops. + The 57 Mbps Moonlight bitrate was too aggressive for the WiFi link''s consistency, even though the average throughput + is ~65 Mbps. + + + ## Mitigations applied + + + ### mac-mini — Moonlight (via `defaults write`) + + + | Setting | Before | After | Rationale | + + ||---------|--------|-------|-----------| + + || bitrate | 57 Mbps | 25 Mbps | Reduce to a level WiFi can sustain consistently | + + || framepacing | off (0) | on (1) | Smooths frame timing over variable latency | + + || fps | 30 | 60 | Restored to game-appropriate value | + + + ### mac-mini — Moonlight (current, wired — 2026-06-04) + + + | Setting | WiFi value | Wired value | Rationale | + + ||---------|-----------|-------------|-----------| + + || bitrate | 60 Mbps | 80 Mbps | Wired 1 GbE can sustain comfortably | + + || framepacing | on (1) | off (0) | Wired latency is consistent, no smoothing needed | + + || fps | 60 | 60 | Unchanged | + + + ### ludo-mini — Sunshine (via SSH) + + + | Setting | WiFi value | Wired value | Rationale | + + ||---------|-----------|-------------|-----------| + + || bitrate | 40 Mbps | 80 Mbps | Wired can handle full bandwidth | + + || fec_percentage | 2 | 5 | Restored to default — wired has no dropout concern | + + || packetsize | 1024 | 1316 | Restored to default (Ethernet MTU) | + + + Sunshine restarted after changes; confirmed active. + + + ## Resolution + + + ### 2026-06-04 — mac-mini wired to LAN + + + mac-mini plugged into Ethernet (en0, 192.168.178.182). This definitively resolves the WiFi jitter that caused streaming + instability. All WiFi-era conservative settings reverted to appropriate wired values on both ends. + + + ## Open questions + + + ~~1. **Ethernet wiring** — mac-mini has an active en0 port. Plugging it into the same switch/router as ludo-mini would + eliminate WiFi jitter entirely. This is the definitive fix.~~ **RESOLVED** — mac-mini now wired via en0. + + 2. **WiFi alternatives** — If wiring is impractical, a WiFi 6 (802.11ax) USB adapter or Thunderbolt-to-Ethernet adapter + would help, but wired Ethernet is the only reliable solution for game streaming. + + 3. **Netbird route** — `192.168.8.0/24` is routed over Netbird `utun100`. If Moonlight ever falls back to that subnet + (ludo-mini''s 192.168.8.133 LAN IP), traffic tunnels unnecessarily. Moonlight correctly uses 192.168.178.181, so this + is not currently an issue. + + + ## Changelog + + + ### 2026-06-04 — mac-mini wired to LAN, configs reverted from WiFi mitigations + + - mac-mini connected via en0 (wired Ethernet, 192.168.178.182), WiFi mitigations no longer needed + + - Moonlight: bitrate 60→80 Mbps, framepacing disabled + + - Sunshine: bitrate 40→80 Mbps, fec_percentage 2→5, packetsize restored to default (1316) + + - First root cause in investigation (Open questions #1) resolved' + about_slugs: + - ws:mac-mini + - host:strong + tags: + - investigation +- slug: investigations/2026-06-06-authentik-session-lifetime + title: 2026-06-06 — Frequent Authentik login prompts fixed (session duration) + date: '2026-06-06' + status: resolved + duration: '' + content: "# 2026-06-06 — Frequent Authentik login prompts fixed (session duration)\n\n## Summary\n\nUser needed to re-authenticate\ + \ to Authentik several times per day. Root cause was the Django session being configured as a **session cookie** (cleared\ + \ on browser close) with a short 24-hour lifetime. Fixed by extending both the session cookie lifetime and the user_login\ + \ stage session duration to 30 days.\n\n## Timeline\n\n- **2026-06-06 ~23:00** — User reports \"having to login to authentik\ + \ several times per day\"\n- **~23:10** — Investigation begins: check Authentik OAuth2 provider settings, proxy session\ + \ table, Django session config\n- **~23:20** — Found `SESSION_EXPIRE_AT_BROWSER_CLOSE=True` — the `authentik_session`\ + \ cookie has no Max-Age, cleared on browser close\n- **~23:25** — Found the `user_login` stage has `session_duration=seconds=0`,\ + \ which calls `session.set_expiry(0)` → `_session_expire_at_browser_close=True`\n- **~23:30** — Applied DB fix: updated\ + \ `authentik_stages_user_login_userloginstage` → `session_duration='days=30'`\n- **~23:35** — Added `AUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE=days=30`\ + \ to `/opt/authentik.env` (maps to `sessions.unauthenticated_age` in Authentik config, setting `SESSION_COOKIE_AGE` to\ + \ 30 days)\n- **~23:40** — Recreated container with `docker compose up -d` (restart doesn't re-read env file)\n- **~23:45**\ + \ — Verified both changes took effect\n\n## Root cause\n\nTwo independent but compounding issues:\n\n1. **`SESSION_EXPIRE_AT_BROWSER_CLOSE=True`**\ + \ — hardcoded in `/authentik/root/settings.py`. Makes the `authentik_session` cookie a session cookie (no `Max-Age`),\ + \ cleared when the browser closes. Cannot be changed via env vars or config files (hardcoded in Python source).\n\n2.\ + \ **`user_login` stage `session_duration=seconds=0`** — the default-authentication-login stage called `session.set_expiry(0)`,\ + \ which in Django sets `_session_expire_at_browser_close=True` on the session object, overriding any session cookie lifetime\ + \ set via `SESSION_COOKIE_AGE`.\n\n3. **`sessions.unauthenticated_age=days=1`** (default) — `SESSION_COOKIE_AGE` was 86,400\ + \ seconds (24 hours), so even with the browser left open, the server-side session data expired after 24 hours with no\ + \ way to extend it (`SESSION_SAVE_EVERY_REQUEST=False`).\n\n### Why forward-auth worked but OAuth2 didn't\n\n- **Forward-auth\ + \ (Caddy-gated services)** — The `authentik_proxy_*` cookie is a *persistent* cookie with 24-hour validity, set by the\ + \ Authentik proxy outpost. It survives browser restart. Caddy's forward-auth validates this cookie directly with the outpost\ + \ API — no Django session needed.\n- **OAuth2 (Gitea, Jellyfin, etc.)** — These services redirect to `auth.hubris.network/application/o/authorize/`,\ + \ which checks the Django session (`authentik_session` cookie). If the browser was closed, this cookie is gone → user\ + \ is redirected to the login form.\n\n## Changes applied\n\n### 1. Database — `authentik_stages_user_login_userloginstage`\n\ + \n```sql\nUPDATE authentik_stages_user_login_userloginstage\nSET session_duration = 'days=30'\nWHERE stage_ptr_id = (\n\ + \ SELECT stage_uuid FROM authentik_flows_stage \n WHERE name = 'default-authentication-login'\n);\n```\n\nThis causes\ + \ the login stage to call `session.set_expiry(timedelta(days=30))`, which:\n- Sets `_session_expire_at_browser_close=False`\ + \ for that session\n- Sets the session cookie `Max-Age` to 30 days (persistent cookie)\n- Sets the server-side session\ + \ expiry to 30 days\n\n### 2. Environment — `AUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE`\n\nAdded to `/opt/authentik.env`:\n\ + \n```\nAUTHENTIK_SESSIONS__UNAUTHENTICATED_AGE=days=30\n```\n\nThis maps to config key `sessions.unauthenticated_age`,\ + \ which Authentik's `settings.py` reads via:\n\n```python\nSESSION_COOKIE_AGE = timedelta_from_string(\n CONFIG.get(\"\ + sessions.unauthenticated_age\", \"days=1\")\n).total_seconds()\n```\n\nResult: `SESSION_COOKIE_AGE` = 2,592,000 seconds\ + \ (30 days). This is a fallback for sessions that don't go through the `user_login` stage or when the stage's explicit\ + \ `set_expiry` doesn't apply.\n\n## Verification\n\n```python\n# Inside authentik-server container:\nimport os\nos.environ.setdefault(\"\ + DJANGO_SETTINGS_MODULE\", \"authentik.root.settings\")\nimport django\ndjango.setup()\nfrom django.conf import settings\n\ + print(\"SESSION_COOKIE_AGE:\", settings.SESSION_COOKIE_AGE) # → 2592000.0\nprint(\"SESSION_EXPIRE_AT_BROWSER_CLOSE:\"\ + , settings.SESSION_EXPIRE_AT_BROWSER_CLOSE) # → True\n\nfrom authentik.stages.user_login.models import UserLoginStage\n\ + stage = UserLoginStage.objects.filter(name=\"default-authentication-login\").first()\nprint(\"session_duration:\", stage.session_duration)\ + \ # → \"days=30\"\n```\n\n## Open questions\n\n- `SESSION_COOKIE_SECURE=False` — Should be `True` since Authentik is\ + \ HTTPS-only, but the custom `SessionMiddleware.is_secure()` method dynamically sets the cookie's `secure` flag based\ + \ on the request, so it's fine.\n- 30 days is generous. Could be reduced to 7-14 days if desired. Change the DB value\ + \ and env var accordingly.\n\n## Related\n\n- [Container 106 — auth-outpost](../../wiki/containers/106-auth-outpost.md)\n\ + - [Authentik VPS migration](archive/2026-05-31-authentik-vps-migration.md)\n- [Ingress (VPS Traefik)](../../wiki/infrastructure/ingress.md)\n\ + - `.hermes/plans/2026-06-06_232200-authentik-frequent-login-fix.md` — original plan\n\n## Changelog\n\n### 2026-06-06\ + \ — created\nDocumented the session lifetime investigation, root cause, and applied fixes." + about_slugs: + - service:caddy + - service:authentik + - service:gitea + - service:jellyfin + - host:hubris + tags: + - investigation +- slug: investigations/2026-06-06-caddyfile-truncation + title: 'Investigation: Caddyfile truncation — all LAN services down (2026-06-06)' + date: '2026-06-06' + status: resolved + duration: ~10 hours (from last known good state ~12:39 UTC to restoration ~22:40 UTC) + content: '# Investigation: Caddyfile truncation — all LAN services down (2026-06-06) + + + **Date:** 2026-06-06 + + **Status:** resolved + + **Duration:** ~10 hours (from last known good state ~12:39 UTC to restoration ~22:40 UTC) + + + ## Symptom + + + All `*.hubris.network` URLs except `photos.hubris.network` and `auth.hubris.network` returned `tlsv1 alert internal error` + or TCP timeouts from LAN/mesh clients. `dig @192.168.8.2` and `dig @100.122.255.254` both resolved to `192.168.8.175` + correctly — DNS was fine. The issue was at the Caddy level. + + + ## Root cause + + + The Caddyfile on LXC 121 was manually edited directly on the filesystem (not via the `dtoro/caddy-conf` git repo), reducing + it from 260 lines/30+ site blocks to 43 lines with only 3 photo-related site blocks: `photos.hubris.network`, `prism.hubris.network`, + and `photos2.hubris.network`. + + + ### Timeline + + + | Time (UTC+2) | Event | + + |---|---| + + | Jun 04 23:43 | Last successful git-push deploy — full Caddyfile (260 lines) | + + | Jun 06 ~12:00 | Caddyfile manually edited locally, truncating to 3 sites | + + | Jun 06 12:39 | Deploy webhook triggered → `git pull --ff-only` failed: "Your local changes would be overwritten" | + + | Jun 06 14:13 | Deploy webhook triggered again → `deploy ok` (the truncated file was committed or merged somehow) | + + | Jun 06 22:34 | Investigation began | + + | Jun 06 22:43 | Caddyfile restored from `origin/master`, `systemctl reload caddy` | + + + ### Evidence + + + - `git diff HEAD -- Caddyfile` on LXC 121: `+3 / -159` lines + + - Git reflog: HEAD at `32575ce` (fix: sab port 8081→8082), working tree diverged + + - Backup file `Caddyfile.bak.1780263919`: 225 lines, full original config + + - `git stash list` shows one auto-stash entry + + - `origin/master` at `1b977aa`: 260 lines, all site blocks present + + + ### Secondary root cause found during investigation + + + **elementsynapse (LXC 118)** had `iface eth0 inet dhcp` internally despite `pct set 118 --net0 ... ip=192.168.8.239/24`. + On DHCP lease renewal, dhclient grabbed `.244` from Technitium''s pool. Caddy''s `reverse_proxy 192.168.8.239:8008` was + hitting a dead IP. + + + This is the same class of drift as the June 5th incidents (paperless, HAOS, apps, mule-images). Elementsynapse was missed + during the 2026-06-02 static-IP migration. + + + ## Fix applied + + + 1. **Caddyfile** → `git checkout --force origin/master -- Caddyfile` + `systemctl reload caddy` + + 2. **elementsynapse** → replaced `iface eth0 inet dhcp` with static, killed dhclient, verified connectivity + + + ## Permanent safeguards (all deployed) + + + | Safeguard | Location | What it does | + + |---|---|---| + + | Site-count guard | `/etc/caddy/scripts/deploy.sh` | Refuses reload if <20 `hubris.network` site blocks | + + | Dirty-tree auto-stash | `/etc/caddy/scripts/deploy.sh` | Stashes local edits before git pull | + + | Auto-backup | `/etc/caddy/scripts/deploy.sh` | Saves Caddyfile.bak. before any change, keeps 5 | + + | Caddy backend health | `/etc/cron.d/caddy-backend-health` on hubris | Runs `check-caddy-backends.sh` every 10 min | + + | DNS sync | `/etc/cron.d/dns-sync` on LXC 107 | Runs `dns-sync.py` every 10 min (was missing since 2026-06-04) | + + + ## Related + + + - DHCP drift investigation (previous incident) — not filed as its own investigation; see the [DNS sync fix](../../../.hermes/plans/2026-06-05_170000-prevent-dhcp-ip-drift.md) + + - [Caddy (121)](../../wiki/containers/121-caddy.md) + + - [elementsynapse (118)](../../wiki/containers/118-elementsynapse.md) + + - [dns-sync script](../../../scripts/dns-sync.py) + + - [check-caddy-backends script](../../../scripts/check-caddy-backends.sh)' + about_slugs: + - service:caddy + - service:dns + - service:paperless + - host:hubris tags: - investigation runbooks: