fix(scheduler): honour check_defs.interval_s, and renumber migrations off main
ListEnabledCheckDefs selected interval_s but never filtered on it, so every enabled check ran on every 30s pass and the declared per-check intervals were decorative. Invisible at 17 enabled checks; at ~180 it would have meant ~126 SSH connections every 30s (~363k/day) and `apt update` on every machine every 30 seconds — 14,400 mirror hits a day to answer a question that changes daily. - check_defs.last_run_at (migration 026) + a due-ness predicate in the query. A column rather than scheduler memory because this control plane restarts on every deploy, and an in-memory map would re-fire every check on each restart. - runCheck stamps last_run_at before processing the result, so a permanently failing check backs off to its interval instead of re-running every pass. - updates and backup-freshness drop to daily. Both answer questions whose answers change about once a day; 60s was just the shared ssh-script default. - last_run_at is seeded to a random offset within the interval so checks created by the same seed do not stay in lockstep — otherwise ~165 probes land in the same instant each minute instead of spread across it. Deliberately not in the upsert's DO UPDATE: a re-seed must not re-herd them. Steady state becomes ~180k SSH/day (down from ~363k) and 5 apt runs/day (down from 14,400), with each 60s check landing at its own point in the minute. Also renumbers 022→023, 023→024, 024→025: origin/main added its own 022_knowledge_revisions, and prod has already applied version 22. Left colliding, prod would have skipped the monitoring_spec migration entirely and then failed the seed on a missing column. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -51,7 +51,7 @@ func (q *Queries) GetAutonomySetting(ctx context.Context, key string) (string, e
|
||||
}
|
||||
|
||||
const getCheckDef = `-- name: GetCheckDef :one
|
||||
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at FROM check_defs WHERE entity_id = $1
|
||||
SELECT entity_id, target_id, target_type, kind, config, interval_s, timeout_s, zone, enabled, updated_at, last_run_at FROM check_defs WHERE entity_id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef, error) {
|
||||
@@ -68,6 +68,7 @@ func (q *Queries) GetCheckDef(ctx context.Context, entityID uuid.UUID) (CheckDef
|
||||
&i.Zone,
|
||||
&i.Enabled,
|
||||
&i.UpdatedAt,
|
||||
&i.LastRunAt,
|
||||
)
|
||||
return i, err
|
||||
}
|
||||
@@ -695,6 +696,8 @@ SELECT cd.entity_id, cd.target_id, cd.target_type, cd.kind, cd.config,
|
||||
FROM check_defs cd
|
||||
JOIN entities e ON e.id = cd.entity_id
|
||||
WHERE cd.enabled = true
|
||||
AND (cd.last_run_at IS NULL
|
||||
OR cd.last_run_at <= now() - make_interval(secs => cd.interval_s))
|
||||
`
|
||||
|
||||
type ListEnabledCheckDefsRow struct {
|
||||
@@ -714,6 +717,9 @@ type ListEnabledCheckDefsRow struct {
|
||||
// =====================================================================
|
||||
// Phase 3 queries
|
||||
// =====================================================================
|
||||
// Enabled AND due. interval_s used to be selected but never filtered on, so
|
||||
// every check ran on every 30s pass and the declared intervals meant nothing.
|
||||
// NULL last_run_at = never run = due now.
|
||||
func (q *Queries) ListEnabledCheckDefs(ctx context.Context) ([]ListEnabledCheckDefsRow, error) {
|
||||
rows, err := q.db.Query(ctx, listEnabledCheckDefs)
|
||||
if err != nil {
|
||||
@@ -1126,6 +1132,15 @@ func (q *Queries) ListSkills(ctx context.Context, status *string) ([]Skill, erro
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const markCheckRun = `-- name: MarkCheckRun :exec
|
||||
UPDATE check_defs SET last_run_at = now() WHERE entity_id = $1
|
||||
`
|
||||
|
||||
func (q *Queries) MarkCheckRun(ctx context.Context, entityID uuid.UUID) error {
|
||||
_, err := q.db.Exec(ctx, markCheckRun, entityID)
|
||||
return err
|
||||
}
|
||||
|
||||
const putIdempotentResponse = `-- name: PutIdempotentResponse :exec
|
||||
INSERT INTO idempotency_keys (actor, key, request_hash, response_code, response_body)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
|
||||
Reference in New Issue
Block a user