sf snapshot: uncommitted changes after 38m inactivity
This commit is contained in:
parent
7b0b346928
commit
d8570d059e
7 changed files with 92 additions and 35 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -29,6 +29,7 @@ Thumbs.db
|
|||
*~
|
||||
.idea/
|
||||
.vscode/
|
||||
.vtcode/
|
||||
*.code-workspace
|
||||
.env
|
||||
.env.*
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
"files": {
|
||||
"includes": [
|
||||
"**/*.{js,cjs,mjs,ts,tsx,json,jsonc,css,html}",
|
||||
"!!.vtcode",
|
||||
"!!.sf",
|
||||
"!!.omg",
|
||||
"!!**/dist",
|
||||
|
|
|
|||
|
|
@ -4115,9 +4115,9 @@ export function getGateCircuitBreaker(gateId) {
|
|||
*/
|
||||
export function updateGateCircuitBreaker(gateId, updates) {
|
||||
if (!currentDb) return;
|
||||
currentDb
|
||||
.prepare(
|
||||
`INSERT INTO gate_circuit_breakers (
|
||||
currentDb
|
||||
.prepare(
|
||||
`INSERT INTO gate_circuit_breakers (
|
||||
gate_id, state, failure_streak, last_failure_at, opened_at, half_open_attempts, updated_at
|
||||
) VALUES (
|
||||
:gate_id, :state, :failure_streak, :last_failure_at, :opened_at, :half_open_attempts, :updated_at
|
||||
|
|
@ -4129,23 +4129,25 @@ export function updateGateCircuitBreaker(gateId, updates) {
|
|||
opened_at = COALESCE(excluded.opened_at, gate_circuit_breakers.opened_at),
|
||||
half_open_attempts = excluded.half_open_attempts,
|
||||
updated_at = excluded.updated_at`,
|
||||
)
|
||||
.run({
|
||||
":gate_id": gateId,
|
||||
":state": updates.state ?? "closed",
|
||||
":failure_streak": updates.failureStreak ?? 0,
|
||||
":last_failure_at": updates.lastFailureAt ?? null,
|
||||
":opened_at": updates.openedAt ?? null,
|
||||
":half_open_attempts": updates.halfOpenAttempts ?? 0,
|
||||
":updated_at": new Date().toISOString(),
|
||||
});
|
||||
)
|
||||
.run({
|
||||
":gate_id": gateId,
|
||||
":state": updates.state ?? "closed",
|
||||
":failure_streak": updates.failureStreak ?? 0,
|
||||
":last_failure_at": updates.lastFailureAt ?? null,
|
||||
":opened_at": updates.openedAt ?? null,
|
||||
":half_open_attempts": updates.halfOpenAttempts ?? 0,
|
||||
":updated_at": new Date().toISOString(),
|
||||
});
|
||||
return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 };
|
||||
}
|
||||
export function getGateLatencyStats(gateId, windowHours = 24) {
|
||||
if (!currentDb) {
|
||||
return { total: 0, avgMs: 0, p50Ms: 0, p95Ms: 0, maxMs: 0 };
|
||||
}
|
||||
const cutoff = new Date(Date.now() - windowHours * 60 * 60 * 1000).toISOString();
|
||||
const cutoff = new Date(
|
||||
Date.now() - windowHours * 60 * 60 * 1000,
|
||||
).toISOString();
|
||||
try {
|
||||
const row = currentDb
|
||||
.prepare(
|
||||
|
|
|
|||
|
|
@ -228,7 +228,11 @@ test("run_records_every_attempt_to_gate_runs", async () => {
|
|||
type: "verification",
|
||||
execute: async () => {
|
||||
calls++;
|
||||
return { outcome: "fail", failureClass: "execution", rationale: `attempt ${calls}` };
|
||||
return {
|
||||
outcome: "fail",
|
||||
failureClass: "execution",
|
||||
rationale: `attempt ${calls}`,
|
||||
};
|
||||
},
|
||||
});
|
||||
await runner.run("audit-gate", makeCtx());
|
||||
|
|
@ -311,6 +315,7 @@ test("circuitBreaker_when_fails_incrementally_opens_after_threshold", async () =
|
|||
assert.equal(breaker.state, "open");
|
||||
assert.equal(breaker.failureStreak, 5);
|
||||
assert.ok(breaker.openedAt != null);
|
||||
assert.equal(calls, 10);
|
||||
});
|
||||
|
||||
test("circuitBreaker_when_open_blocks_execution", async () => {
|
||||
|
|
@ -366,7 +371,11 @@ test("circuitBreaker_half_open_fail_reopens", async () => {
|
|||
runner.register({
|
||||
id: "cb-reopen",
|
||||
type: "verification",
|
||||
execute: async () => ({ outcome: "fail", failureClass: "execution", rationale: "nope" }),
|
||||
execute: async () => ({
|
||||
outcome: "fail",
|
||||
failureClass: "execution",
|
||||
rationale: "nope",
|
||||
}),
|
||||
});
|
||||
updateGateCircuitBreaker("cb-reopen", {
|
||||
state: "half-open",
|
||||
|
|
@ -425,7 +434,11 @@ test("validateGate_when_missing_id_returns_false", () => {
|
|||
});
|
||||
|
||||
test("validateGate_when_empty_id_returns_false", () => {
|
||||
const result = validateGate({ id: "", type: "policy", execute: async () => ({}) });
|
||||
const result = validateGate({
|
||||
id: "",
|
||||
type: "policy",
|
||||
execute: async () => ({}),
|
||||
});
|
||||
assert.equal(result.valid, false);
|
||||
assert.ok(result.reason.includes("id"));
|
||||
});
|
||||
|
|
@ -450,8 +463,15 @@ test("validateGate_when_execute_not_function_returns_false", () => {
|
|||
|
||||
test("runner_register_when_invalid_gate_throws", () => {
|
||||
const runner = new UokGateRunner();
|
||||
assert.throws(() => runner.register({ id: "", type: "policy", execute: async () => ({}) }), /id/);
|
||||
assert.throws(() => runner.register({ id: "x", type: "", execute: async () => ({}) }), /type/);
|
||||
assert.throws(
|
||||
() =>
|
||||
runner.register({ id: "", type: "policy", execute: async () => ({}) }),
|
||||
/id/,
|
||||
);
|
||||
assert.throws(
|
||||
() => runner.register({ id: "x", type: "", execute: async () => ({}) }),
|
||||
/type/,
|
||||
);
|
||||
assert.throws(() => runner.register({ id: "x", type: "policy" }), /execute/);
|
||||
assert.throws(() => runner.register(null), /object/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -160,6 +160,16 @@ test("decide_when_stale_with_budget_returns_retry", () => {
|
|||
assert.equal(d.action, "retry");
|
||||
});
|
||||
|
||||
test("decide_when_runaway_recovered_requires_explicit_reset", () => {
|
||||
const d = decideUnitRuntimeDispatch({
|
||||
status: "runaway-recovered",
|
||||
retryCount: 0,
|
||||
maxRetries: 1,
|
||||
});
|
||||
assert.equal(d.action, "block");
|
||||
assert.equal(d.reasonCode, "runaway-recovery-reset-required");
|
||||
});
|
||||
|
||||
test("decide_when_blocked_returns_notify", () => {
|
||||
const d = decideUnitRuntimeDispatch({ status: "blocked" });
|
||||
assert.equal(d.action, "notify");
|
||||
|
|
|
|||
|
|
@ -41,16 +41,32 @@ function collectGateMetrics() {
|
|||
const lines = [];
|
||||
for (const gateId of GATE_NAMES) {
|
||||
const stats = getGateRunStats(gateId, 24);
|
||||
lines.push(fmtCounter("uok_gate_runs_total", stats.total, { gate_id: gateId }));
|
||||
lines.push(fmtCounter("uok_gate_runs_passed_total", stats.pass, { gate_id: gateId }));
|
||||
lines.push(fmtCounter("uok_gate_runs_failed_total", stats.fail, { gate_id: gateId }));
|
||||
lines.push(fmtCounter("uok_gate_runs_retry_total", stats.retry, { gate_id: gateId }));
|
||||
lines.push(
|
||||
fmtCounter("uok_gate_runs_total", stats.total, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtCounter("uok_gate_runs_passed_total", stats.pass, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtCounter("uok_gate_runs_failed_total", stats.fail, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtCounter("uok_gate_runs_retry_total", stats.retry, { gate_id: gateId }),
|
||||
);
|
||||
|
||||
const latency = getGateLatencyStats(gateId, 24);
|
||||
lines.push(fmtGauge("uok_gate_latency_avg_ms", latency.avgMs, { gate_id: gateId }));
|
||||
lines.push(fmtGauge("uok_gate_latency_p50_ms", latency.p50Ms, { gate_id: gateId }));
|
||||
lines.push(fmtGauge("uok_gate_latency_p95_ms", latency.p95Ms, { gate_id: gateId }));
|
||||
lines.push(fmtGauge("uok_gate_latency_max_ms", latency.maxMs, { gate_id: gateId }));
|
||||
lines.push(
|
||||
fmtGauge("uok_gate_latency_avg_ms", latency.avgMs, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtGauge("uok_gate_latency_p50_ms", latency.p50Ms, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtGauge("uok_gate_latency_p95_ms", latency.p95Ms, { gate_id: gateId }),
|
||||
);
|
||||
lines.push(
|
||||
fmtGauge("uok_gate_latency_max_ms", latency.maxMs, { gate_id: gateId }),
|
||||
);
|
||||
|
||||
const breaker = getGateCircuitBreaker(gateId);
|
||||
const stateMap = { closed: 0, "half-open": 1, open: 2 };
|
||||
|
|
@ -61,9 +77,13 @@ function collectGateMetrics() {
|
|||
}),
|
||||
);
|
||||
lines.push(
|
||||
fmtGauge("uok_gate_circuit_breaker_failure_streak", breaker.failureStreak, {
|
||||
gate_id: gateId,
|
||||
}),
|
||||
fmtGauge(
|
||||
"uok_gate_circuit_breaker_failure_streak",
|
||||
breaker.failureStreak,
|
||||
{
|
||||
gate_id: gateId,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
return lines;
|
||||
|
|
|
|||
|
|
@ -98,11 +98,7 @@ export const UNIT_RUNTIME_TRANSITIONS = {
|
|||
notified: ["queued"],
|
||||
};
|
||||
const DEFAULT_UNIT_RUNTIME_MAX_RETRIES = 1;
|
||||
const RETRYABLE_TERMINAL_STATUSES = new Set([
|
||||
"failed",
|
||||
"stale",
|
||||
"runaway-recovered",
|
||||
]);
|
||||
const RETRYABLE_TERMINAL_STATUSES = new Set(["failed", "stale"]);
|
||||
function hasUpdate(updates, key) {
|
||||
return Object.hasOwn(updates, key);
|
||||
}
|
||||
|
|
@ -242,6 +238,13 @@ export function decideUnitRuntimeDispatch(record, options = {}) {
|
|||
...common,
|
||||
};
|
||||
}
|
||||
if (state.status === "runaway-recovered") {
|
||||
return {
|
||||
action: "block",
|
||||
reasonCode: "runaway-recovery-reset-required",
|
||||
...common,
|
||||
};
|
||||
}
|
||||
if (RETRYABLE_TERMINAL_STATUSES.has(state.status)) {
|
||||
if (remaining > 0) {
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue