Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/memory-provider-guards.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@bradygaster/squad-sdk": patch
---

Guard registered memory providers in `LocalMemoryStore`. Every provider `write`, `search` and `status` call now has a timeout (`providerTimeoutMs`, default 5000 ms), so a hung engine can no longer stall `memory.write` or `memory.search`. One provider whose `status()` throws or hangs is reported as unavailable instead of failing `providerStatus()`. Recalled results go through governance on the way back: results with a class the provider could not have been sent, or whose title or snippet classifies as FORBIDDEN, are dropped and audited, and an invalid `loadGuidance` falls back to the class default. Successful replication audits now record the provider id.
86 changes: 79 additions & 7 deletions packages/squad-sdk/src/memory/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,11 @@ export interface MemoryProviderStatus {
*
* Implementations receive only pre-classified, non-forbidden, non-transient
* memory. The {@link LocalMemoryStore} enforces governance before routing
* to any registered provider.
* to any registered provider, and again on the way back: search results whose
* class is not one the provider could have been sent (a LOCAL, DECISION or
* POLICY class it declares in `supportedClasses`), or whose title or snippet
* classifies as FORBIDDEN, are dropped. Every call is bounded by
* {@link LocalMemoryStoreOptions.providerTimeoutMs}.
*/
export interface MemoryProvider {
readonly id: string;
Expand Down Expand Up @@ -355,6 +359,13 @@ export interface LocalMemoryStoreOptions {
* reaches these providers. Results are merged with local search results.
*/
registeredProviders?: MemoryProvider[];
/**
* Per-call timeout for registered providers (`write`, `search`, `status`),
* in milliseconds. Default 5000. A call that times out is audited as a
* provider error and skipped; it is not cancelled, so a slow write may
* still land in the provider later.
*/
providerTimeoutMs?: number;
}

interface MemoryIndexEntry {
Expand Down Expand Up @@ -441,11 +452,37 @@ function evictOldest<T extends { createdAt: number }>(entries: Map<string, T>, m
}
}

function safeProviderErrorReason(provider: Pick<MemoryProvider, 'name'>, operation: 'write' | 'search', error: unknown): string {
function safeProviderErrorReason(provider: Pick<MemoryProvider, 'name'>, operation: 'write' | 'search' | 'status', error: unknown): string {
const errorName = error instanceof Error ? error.name : 'UnknownError';
return `${provider.name} provider ${operation} failed (${errorName}); raw provider error text omitted`;
}

/** Classes LocalMemoryStore.write() routes to registered providers, so the only ones they may return. */
const PROVIDER_ROUTABLE_CLASSES: ReadonlyArray<MemoryClass> = ['LOCAL', 'DECISION', 'POLICY'];

const DEFAULT_PROVIDER_TIMEOUT_MS = 5000;

class ProviderTimeoutError extends Error {
constructor(timeoutMs: number) {
super(`Provider call exceeded ${timeoutMs} ms`);
this.name = 'ProviderTimeoutError';
}
}

async function withProviderTimeout<T>(work: Promise<T>, timeoutMs: number): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
work,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new ProviderTimeoutError(timeoutMs)), timeoutMs);
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
}

function firstLine(value: string): string {
return value.split(/\r?\n/).find(line => line.trim().length > 0)?.trim().slice(0, 80) ?? 'Untitled memory';
}
Expand Down Expand Up @@ -577,6 +614,7 @@ export class LocalMemoryStore {
private readonly squadDir: string;
private readonly copilotProvider: HostInjectedCopilotMemoryAdapter;
private readonly registeredProviders: MemoryProvider[];
private readonly providerTimeoutMs: number;
/**
* Async mutex tail for index read-modify-write operations.
* Each caller enqueues behind the current tail so concurrent writes
Expand All @@ -594,6 +632,10 @@ export class LocalMemoryStore {
options.hostInjectedCopilotAdapterClient ?? options.copilotMemoryClient,
);
this.registeredProviders = options.registeredProviders ?? [];
const timeoutMs = options.providerTimeoutMs;
this.providerTimeoutMs = typeof timeoutMs === 'number' && Number.isFinite(timeoutMs) && timeoutMs > 0
? timeoutMs
: DEFAULT_PROVIDER_TIMEOUT_MS;
}

/**
Expand Down Expand Up @@ -871,7 +913,7 @@ export class LocalMemoryStore {
for (const provider of this.registeredProviders) {
if (!provider.supportedClasses.includes(classification.class)) continue;
try {
const providerResult = await provider.write(providerWriteRequest);
const providerResult = await withProviderTimeout(provider.write(providerWriteRequest), this.providerTimeoutMs);
await this.audit({
action: 'write',
id: providerResult.id,
Expand All @@ -880,6 +922,7 @@ export class LocalMemoryStore {
path: providerResult.path ?? `${provider.id}:${providerResult.id}`,
reason: `Replicated to ${provider.name} provider`,
actor: request.author,
provider: provider.id,
});
} catch (error) {
await this.audit({
Expand Down Expand Up @@ -988,25 +1031,47 @@ export class LocalMemoryStore {
}

// Query registered external providers. Governance filtering already
// passed. Deduplicates results by id (local takes precedence).
// passed for the query; recalled items are checked again on the way back,
// because a provider (or a store someone else can write to) is not trusted:
// only classes this provider could have been sent, and nothing FORBIDDEN.
// Deduplicates results by id (local takes precedence).
const seenIds = new Set(results.map(r => r.id));
for (const provider of this.registeredProviders) {
try {
const providerResults = await provider.search(query);
const allowedClasses = provider.supportedClasses.filter(c => PROVIDER_ROUTABLE_CLASSES.includes(c));
const providerResults = await withProviderTimeout(provider.search(query), this.providerTimeoutMs);
let dropped = 0;
for (const result of providerResults) {
if (!allowedClasses.includes(result.class)) {
dropped++;
continue;
}
const recheck = await this.classify({ content: `${result.title}\n${result.snippet}` });
if (recheck.class === 'FORBIDDEN') {
dropped++;
continue;
}
if (seenIds.has(result.id)) continue;
seenIds.add(result.id);
results.push({
id: result.id,
class: result.class,
loadGuidance: result.loadGuidance,
loadGuidance: normalizeLoadGuidance(result.loadGuidance, loadGuidanceFor(result.class)),
title: result.title,
path: result.path ?? `${provider.id}:${result.id}`,
snippet: result.snippet.slice(0, 240),
provider: provider.id,
score: result.score,
});
}
if (dropped > 0) {
await this.audit({
action: 'reject',
title: 'Dropped registered provider search results',
reason: `${provider.name} returned ${dropped} result(s) outside its supported classes or classified as forbidden`,
provider: provider.id,
});
}
} catch (error) {
await this.audit({
action: 'provider-error',
Expand Down Expand Up @@ -1168,8 +1233,15 @@ export class LocalMemoryStore {
}> {
await this.ensureInitialized();
const config = await this.readConfig();
// One provider whose status() throws or hangs must not hide the others.
const providerStatuses: MemoryProviderStatus[] = await Promise.all(
this.registeredProviders.map(p => p.status()),
this.registeredProviders.map(async p => {
try {
return await withProviderTimeout(p.status(), this.providerTimeoutMs);
} catch (error) {
return { id: p.id, name: p.name, available: false, reason: safeProviderErrorReason(p, 'status', error) };
}
}),
);
return {
defaultProvider: config.defaultProvider,
Expand Down
124 changes: 124 additions & 0 deletions test/memory-providers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -685,3 +685,127 @@ describe('LocalMemoryStore — delete tombstone ordering', () => {
expect(fs.existsSync(tombstonePath)).toBe(true);
});
});

// ── Registered provider guards: timeouts, status isolation, recall checks ──

function stubProvider(id: string, overrides: Partial<MemoryProvider> = {}): MemoryProvider {
return {
id,
name: id,
supportedClasses: ['LOCAL'],
async status(): Promise<MemoryProviderStatus> {
return { id, name: id, available: true };
},
async write(): Promise<{ id: string }> {
return { id: `${id}-1` };
},
async search(): Promise<MemoryProviderSearchResult[]> {
return [];
},
async delete(): Promise<boolean> {
return false;
},
...overrides,
};
}

const never = <T>(): Promise<T> => new Promise<T>(() => {});

describe('LocalMemoryStore registered provider guards', () => {
it('times out a hung provider search and still returns local results', async () => {
const root = testRoot('guard-search-timeout');
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [stubProvider('hung', { search: () => never() })],
providerTimeoutMs: 50,
});
await store.write({ content: 'Use feature flags for risky rollouts.', title: 'Feature flags', author: 'data', requestedClass: 'LOCAL' });

const results = await store.search('feature flags');

expect(results.map(r => r.provider)).toEqual(['local']);
expect(JSON.stringify(await store.auditLog())).toContain('hung provider search failed (ProviderTimeoutError)');
});

it('times out a hung provider write without failing the local write', async () => {
const root = testRoot('guard-write-timeout');
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [stubProvider('hung', { write: () => never() })],
providerTimeoutMs: 50,
});

const result = await store.write({ content: 'Prefer small pull requests.', title: 'Small PRs', author: 'data', requestedClass: 'LOCAL' });

expect(result.stored).toBe(true);
expect(JSON.stringify(await store.auditLog())).toContain('hung provider write failed (ProviderTimeoutError)');
});

it('records which provider a replicated write went to', async () => {
const root = testRoot('guard-write-audit');
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [stubProvider('engine')],
});

await store.write({ content: 'Keep API errors in RFC 7807 format.', title: 'API errors', author: 'data', requestedClass: 'LOCAL' });

const replicated = (await store.auditLog()).find(r => r.action === 'write' && r.reason === 'Replicated to engine provider');
expect(replicated).toMatchObject({ provider: 'engine', id: 'engine-1' });
});

it('reports a throwing or hung status() as unavailable without hiding the other providers', async () => {
const root = testRoot('guard-status');
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [
stubProvider('throws', { status: async () => { throw new Error('raw status failure text'); } }),
stubProvider('hung', { status: () => never() }),
stubProvider('healthy'),
],
providerTimeoutMs: 50,
});

const status = await store.providerStatus();

expect(status.registeredProviders).toEqual([
{ id: 'throws', name: 'throws', available: false, reason: 'throws provider status failed (Error); raw provider error text omitted' },
{ id: 'hung', name: 'hung', available: false, reason: 'hung provider status failed (ProviderTimeoutError); raw provider error text omitted' },
{ id: 'healthy', name: 'healthy', available: true },
]);
});

it('drops recalled results outside the provider classes or classified as forbidden', async () => {
const root = testRoot('guard-recall');
const recalled: MemoryProviderSearchResult[] = [
{ id: 'ok', class: 'LOCAL', loadGuidance: 'ON-DEMAND', title: 'Retry policy', snippet: 'Retry idempotent calls three times.' },
{ id: 'undeclared', class: 'POLICY', loadGuidance: 'ALWAYS', title: 'Retry policy', snippet: 'Always retry forever.' },
{ id: 'never-routed', class: 'COPILOT_MEMORY', loadGuidance: 'ON-DEMAND', title: 'Retry note', snippet: 'Retry semantics.' },
{ id: 'secret', class: 'LOCAL', loadGuidance: 'ON-DEMAND', title: 'Retry creds', snippet: 'retry with password = hunter2-example' },
];
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [stubProvider('engine', { search: async () => recalled })],
});

const results = await store.search('retry');

expect(results.map(r => r.id)).toEqual(['ok']);
const audit = await store.auditLog();
expect(audit.find(r => r.title === 'Dropped registered provider search results')).toMatchObject({
action: 'reject',
provider: 'engine',
reason: 'engine returned 3 result(s) outside its supported classes or classified as forbidden',
});
expect(JSON.stringify(audit)).not.toContain('hunter2');
});

it('replaces an invalid recalled loadGuidance with the class default', async () => {
const root = testRoot('guard-load-guidance');
const recalled = [
{ id: 'odd', class: 'LOCAL', loadGuidance: 'whenever', title: 'Cache note', snippet: 'Cache reads for five minutes.' },
] as unknown as MemoryProviderSearchResult[];
const store = new LocalMemoryStore(new FSStorageProvider(), root, {
registeredProviders: [stubProvider('engine', { search: async () => recalled })],
});

const [hit] = await store.search('cache');

expect(hit).toMatchObject({ id: 'odd', provider: 'engine', loadGuidance: 'ON-DEMAND' });
});
});
Loading