yuanyeex
/
openclaw
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix build regressions after merge

main
Benjamin Jesuiter 2026-02-02 22:47:11 +01:00 committed by Vignesh
parent 465536e811
commit 23cfcd60df
3 changed files with 360 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/config/zod-schema.ts
View File

@ -87,7 +87,7 @@ const MemorySchema = z
.strict() .strict()
.optional(); .optional();
export const MoltbotSchema = z export const OpenClawSchema = z
.object({ .object({
meta: z meta: z
.object({ .object({

499
src/memory/manager.ts
View File

@ -1,50 +1,11 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite"; import type { DatabaseSync } from "node:sqlite";
import chokidar, { type FSWatcher } from "chokidar"; import chokidar, { type FSWatcher } from "chokidar";
import { randomUUID } from "node:crypto";
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js"; import type { ResolvedMemorySearchConfig } from "../agents/memory-search.js";
import { resolveMemorySearchConfig } from "../agents/memory-search.js"; import type { OpenClawConfig } from "../config/config.js";
import type { MoltbotConfig } from "../config/config.js";
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
import { resolveUserPath } from "../utils.js";
import {
createEmbeddingProvider,
type EmbeddingProvider,
type EmbeddingProviderResult,
type GeminiEmbeddingClient,
type OpenAiEmbeddingClient,
} from "./embeddings.js";
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
import {
OPENAI_BATCH_ENDPOINT,
type OpenAiBatchRequest,
runOpenAiEmbeddingBatches,
} from "./batch-openai.js";
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
import {
buildFileEntry,
chunkMarkdown,
ensureDir,
hashText,
isMemoryPath,
listMemoryFiles,
type MemoryChunk,
type MemoryFileEntry,
normalizeRelPath,
parseEmbedding,
} from "./internal.js";
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
import { searchKeyword, searchVector } from "./manager-search.js";
import { ensureMemoryIndexSchema } from "./memory-schema.js";
import { requireNodeSqlite } from "./sqlite.js";
import { loadSqliteVecExtension } from "./sqlite-vec.js";
import type { import type {
MemoryEmbeddingProbeResult, MemoryEmbeddingProbeResult,
MemoryProviderStatus, MemoryProviderStatus,
@ -53,6 +14,44 @@ import type {
MemorySource, MemorySource,
MemorySyncProgressUpdate, MemorySyncProgressUpdate,
} from "./types.js"; } from "./types.js";
import { resolveAgentDir, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import { resolveMemorySearchConfig } from "../agents/memory-search.js";
import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { onSessionTranscriptUpdate } from "../sessions/transcript-events.js";
import { resolveUserPath } from "../utils.js";
import { runGeminiEmbeddingBatches, type GeminiBatchRequest } from "./batch-gemini.js";
import {
OPENAI_BATCH_ENDPOINT,
type OpenAiBatchRequest,
runOpenAiEmbeddingBatches,
} from "./batch-openai.js";
import { DEFAULT_GEMINI_EMBEDDING_MODEL } from "./embeddings-gemini.js";
import { DEFAULT_OPENAI_EMBEDDING_MODEL } from "./embeddings-openai.js";
import {
createEmbeddingProvider,
type EmbeddingProvider,
type EmbeddingProviderResult,
type GeminiEmbeddingClient,
type OpenAiEmbeddingClient,
} from "./embeddings.js";
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
import {
buildFileEntry,
chunkMarkdown,
ensureDir,
hashText,
isMemoryPath,
listMemoryFiles,
normalizeExtraMemoryPaths,
type MemoryChunk,
type MemoryFileEntry,
parseEmbedding,
} from "./internal.js";
import { searchKeyword, searchVector } from "./manager-search.js";
import { ensureMemoryIndexSchema } from "./memory-schema.js";
import { loadSqliteVecExtension } from "./sqlite-vec.js";
import { requireNodeSqlite } from "./sqlite.js";
type MemoryIndexMeta = { type MemoryIndexMeta = {
model: string; model: string;
@ -108,7 +107,7 @@ const vectorToBlob = (embedding: number[]): Buffer =>
export class MemoryIndexManager implements MemorySearchManager { export class MemoryIndexManager implements MemorySearchManager {
private readonly cacheKey: string; private readonly cacheKey: string;
private readonly cfg: MoltbotConfig; private readonly cfg: OpenClawConfig;
private readonly agentId: string; private readonly agentId: string;
private readonly workspaceDir: string; private readonly workspaceDir: string;
private readonly settings: ResolvedMemorySearchConfig; private readonly settings: ResolvedMemorySearchConfig;
@ -164,16 +163,20 @@ export class MemoryIndexManager implements MemorySearchManager {
private syncing: Promise<void> | null = null; private syncing: Promise<void> | null = null;
static async get(params: { static async get(params: {
cfg: MoltbotConfig; cfg: OpenClawConfig;
agentId: string; agentId: string;
}): Promise<MemoryIndexManager | null> { }): Promise<MemoryIndexManager | null> {
const { cfg, agentId } = params; const { cfg, agentId } = params;
const settings = resolveMemorySearchConfig(cfg, agentId); const settings = resolveMemorySearchConfig(cfg, agentId);
if (!settings) return null; if (!settings) {
return null;
}
const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId);
const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`; const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`;
const existing = INDEX_CACHE.get(key); const existing = INDEX_CACHE.get(key);
if (existing) return existing; if (existing) {
return existing;
}
const providerResult = await createEmbeddingProvider({ const providerResult = await createEmbeddingProvider({
config: cfg, config: cfg,
agentDir: resolveAgentDir(cfg, agentId), agentDir: resolveAgentDir(cfg, agentId),
@ -197,7 +200,7 @@ export class MemoryIndexManager implements MemorySearchManager {
private constructor(params: { private constructor(params: {
cacheKey: string; cacheKey: string;
cfg: MoltbotConfig; cfg: OpenClawConfig;
agentId: string; agentId: string;
workspaceDir: string; workspaceDir: string;
settings: ResolvedMemorySearchConfig; settings: ResolvedMemorySearchConfig;
@ -240,13 +243,19 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
async warmSession(sessionKey?: string): Promise<void> { async warmSession(sessionKey?: string): Promise<void> {
if (!this.settings.sync.onSessionStart) return; if (!this.settings.sync.onSessionStart) {
return;
}
const key = sessionKey?.trim() || ""; const key = sessionKey?.trim() || "";
if (key && this.sessionWarm.has(key)) return; if (key && this.sessionWarm.has(key)) {
return;
}
void this.sync({ reason: "session-start" }).catch((err) => { void this.sync({ reason: "session-start" }).catch((err) => {
log.warn(`memory sync failed (session-start): ${String(err)}`); log.warn(`memory sync failed (session-start): ${String(err)}`);
}); });
if (key) this.sessionWarm.add(key); if (key) {
this.sessionWarm.add(key);
}
} }
async search( async search(
@ -264,7 +273,9 @@ export class MemoryIndexManager implements MemorySearchManager {
}); });
} }
const cleaned = query.trim(); const cleaned = query.trim();
if (!cleaned) return []; if (!cleaned) {
return [];
}
const minScore = opts?.minScore ?? this.settings.query.minScore; const minScore = opts?.minScore ?? this.settings.query.minScore;
const maxResults = opts?.maxResults ?? this.settings.query.maxResults; const maxResults = opts?.maxResults ?? this.settings.query.maxResults;
const hybrid = this.settings.query.hybrid; const hybrid = this.settings.query.hybrid;
@ -323,7 +334,9 @@ export class MemoryIndexManager implements MemorySearchManager {
query: string, query: string,
limit: number, limit: number,
): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> { ): Promise<Array<MemorySearchResult & { id: string; textScore: number }>> {
if (!this.fts.enabled || !this.fts.available) return []; if (!this.fts.enabled || !this.fts.available) {
return [];
}
const sourceFilter = this.buildSourceFilter(); const sourceFilter = this.buildSourceFilter();
const results = await searchKeyword({ const results = await searchKeyword({
db: this.db, db: this.db,
@ -375,7 +388,9 @@ export class MemoryIndexManager implements MemorySearchManager {
force?: boolean; force?: boolean;
progress?: (update: MemorySyncProgressUpdate) => void; progress?: (update: MemorySyncProgressUpdate) => void;
}): Promise<void> { }): Promise<void> {
if (this.syncing) return this.syncing; if (this.syncing) {
return this.syncing;
}
this.syncing = this.runSync(params).finally(() => { this.syncing = this.runSync(params).finally(() => {
this.syncing = null; this.syncing = null;
}); });
@ -387,13 +402,54 @@ export class MemoryIndexManager implements MemorySearchManager {
from?: number; from?: number;
lines?: number; lines?: number;
}): Promise<{ text: string; path: string }> { }): Promise<{ text: string; path: string }> {
const relPath = normalizeRelPath(params.relPath); const rawPath = params.relPath.trim();
if (!relPath || !isMemoryPath(relPath)) { if (!rawPath) {
throw new Error("path required"); throw new Error("path required");
} }
const absPath = path.resolve(this.workspaceDir, relPath); const absPath = path.isAbsolute(rawPath)
if (!absPath.startsWith(this.workspaceDir)) { ? path.resolve(rawPath)
throw new Error("path escapes workspace"); : path.resolve(this.workspaceDir, rawPath);
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
const inWorkspace =
relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
let allowedAdditional = false;
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
const additionalPaths = normalizeExtraMemoryPaths(
this.workspaceDir,
this.settings.extraPaths,
);
for (const additionalPath of additionalPaths) {
try {
const stat = await fs.lstat(additionalPath);
if (stat.isSymbolicLink()) {
continue;
}
if (stat.isDirectory()) {
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
allowedAdditional = true;
break;
}
continue;
}
if (stat.isFile()) {
if (absPath === additionalPath && absPath.endsWith(".md")) {
allowedAdditional = true;
break;
}
}
} catch {}
}
}
if (!allowedWorkspace && !allowedAdditional) {
throw new Error("path required");
}
if (!absPath.endsWith(".md")) {
throw new Error("path required");
}
const stat = await fs.lstat(absPath);
if (stat.isSymbolicLink() || !stat.isFile()) {
throw new Error("path required");
} }
const content = await fs.readFile(absPath, "utf-8"); const content = await fs.readFile(absPath, "utf-8");
if (!params.from && !params.lines) { if (!params.from && !params.lines) {
@ -420,7 +476,9 @@ export class MemoryIndexManager implements MemorySearchManager {
}; };
const sourceCounts = (() => { const sourceCounts = (() => {
const sources = Array.from(this.sources); const sources = Array.from(this.sources);
if (sources.length === 0) return []; if (sources.length === 0) {
return [];
}
const bySource = new Map<MemorySource, { files: number; chunks: number }>(); const bySource = new Map<MemorySource, { files: number; chunks: number }>();
for (const source of sources) { for (const source of sources) {
bySource.set(source, { files: 0, chunks: 0 }); bySource.set(source, { files: 0, chunks: 0 });
@ -445,7 +503,7 @@ export class MemoryIndexManager implements MemorySearchManager {
entry.chunks = row.c ?? 0; entry.chunks = row.c ?? 0;
bySource.set(row.source, entry); bySource.set(row.source, entry);
} }
return sources.map((source) => ({ source, ...bySource.get(source)! })); return sources.map((source) => Object.assign({ source }, bySource.get(source)!));
})(); })();
return { return {
backend: "builtin", backend: "builtin",
@ -458,6 +516,7 @@ export class MemoryIndexManager implements MemorySearchManager {
model: this.provider.model, model: this.provider.model,
requestedProvider: this.requestedProvider, requestedProvider: this.requestedProvider,
sources: Array.from(this.sources), sources: Array.from(this.sources),
extraPaths: this.settings.extraPaths,
sourceCounts, sourceCounts,
cache: this.cache.enabled cache: this.cache.enabled
? { ? {
@ -501,7 +560,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
async probeVectorAvailability(): Promise<boolean> { async probeVectorAvailability(): Promise<boolean> {
if (!this.vector.enabled) return false; if (!this.vector.enabled) {
return false;
}
return this.ensureVectorReady(); return this.ensureVectorReady();
} }
@ -516,7 +577,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
async close(): Promise<void> { async close(): Promise<void> {
if (this.closed) return; if (this.closed) {
return;
}
this.closed = true; this.closed = true;
if (this.watchTimer) { if (this.watchTimer) {
clearTimeout(this.watchTimer); clearTimeout(this.watchTimer);
@ -543,7 +606,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async ensureVectorReady(dimensions?: number): Promise<boolean> { private async ensureVectorReady(dimensions?: number): Promise<boolean> {
if (!this.vector.enabled) return false; if (!this.vector.enabled) {
return false;
}
if (!this.vectorReady) { if (!this.vectorReady) {
this.vectorReady = this.withTimeout( this.vectorReady = this.withTimeout(
this.loadVectorExtension(), this.loadVectorExtension(),
@ -569,7 +634,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async loadVectorExtension(): Promise<boolean> { private async loadVectorExtension(): Promise<boolean> {
if (this.vector.available !== null) return this.vector.available; if (this.vector.available !== null) {
return this.vector.available;
}
if (!this.vector.enabled) { if (!this.vector.enabled) {
this.vector.available = false; this.vector.available = false;
return false; return false;
@ -579,7 +646,9 @@ export class MemoryIndexManager implements MemorySearchManager {
? resolveUserPath(this.vector.extensionPath) ? resolveUserPath(this.vector.extensionPath)
: undefined; : undefined;
const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath }); const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath });
if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error"); if (!loaded.ok) {
throw new Error(loaded.error ?? "unknown sqlite-vec load error");
}
this.vector.extensionPath = loaded.extensionPath; this.vector.extensionPath = loaded.extensionPath;
this.vector.available = true; this.vector.available = true;
return true; return true;
@ -593,7 +662,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private ensureVectorTable(dimensions: number): void { private ensureVectorTable(dimensions: number): void {
if (this.vector.dims === dimensions) return; if (this.vector.dims === dimensions) {
return;
}
if (this.vector.dims && this.vector.dims !== dimensions) { if (this.vector.dims && this.vector.dims !== dimensions) {
this.dropVectorTable(); this.dropVectorTable();
} }
@ -617,7 +688,9 @@ export class MemoryIndexManager implements MemorySearchManager {
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } { private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
const sources = Array.from(this.sources); const sources = Array.from(this.sources);
if (sources.length === 0) return { sql: "", params: [] }; if (sources.length === 0) {
return { sql: "", params: [] };
}
const column = alias ? `${alias}.source` : "source"; const column = alias ? `${alias}.source` : "source";
const placeholders = sources.map(() => "?").join(", "); const placeholders = sources.map(() => "?").join(", ");
return { sql: ` AND ${column} IN (${placeholders})`, params: sources }; return { sql: ` AND ${column} IN (${placeholders})`, params: sources };
@ -636,7 +709,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private seedEmbeddingCache(sourceDb: DatabaseSync): void { private seedEmbeddingCache(sourceDb: DatabaseSync): void {
if (!this.cache.enabled) return; if (!this.cache.enabled) {
return;
}
try { try {
const rows = sourceDb const rows = sourceDb
.prepare( .prepare(
@ -651,7 +726,9 @@ export class MemoryIndexManager implements MemorySearchManager {
dims: number | null; dims: number | null;
updated_at: number; updated_at: number;
}>; }>;
if (!rows.length) return; if (!rows.length) {
return;
}
const insert = this.db.prepare( const insert = this.db.prepare(
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at) `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
@ -728,12 +805,26 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private ensureWatcher() { private ensureWatcher() {
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return; if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) {
const watchPaths = [ return;
}
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths)
.map((entry) => {
try {
const stat = fsSync.lstatSync(entry);
return stat.isSymbolicLink() ? null : entry;
} catch {
return null;
}
})
.filter((entry): entry is string => Boolean(entry));
const watchPaths = new Set<string>([
path.join(this.workspaceDir, "MEMORY.md"), path.join(this.workspaceDir, "MEMORY.md"),
path.join(this.workspaceDir, "memory.md"),
path.join(this.workspaceDir, "memory"), path.join(this.workspaceDir, "memory"),
]; ...additionalPaths,
this.watcher = chokidar.watch(watchPaths, { ]);
this.watcher = chokidar.watch(Array.from(watchPaths), {
ignoreInitial: true, ignoreInitial: true,
awaitWriteFinish: { awaitWriteFinish: {
stabilityThreshold: this.settings.sync.watchDebounceMs, stabilityThreshold: this.settings.sync.watchDebounceMs,
@ -750,18 +841,26 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private ensureSessionListener() { private ensureSessionListener() {
if (!this.sources.has("sessions") || this.sessionUnsubscribe) return; if (!this.sources.has("sessions") || this.sessionUnsubscribe) {
return;
}
this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => { this.sessionUnsubscribe = onSessionTranscriptUpdate((update) => {
if (this.closed) return; if (this.closed) {
return;
}
const sessionFile = update.sessionFile; const sessionFile = update.sessionFile;
if (!this.isSessionFileForAgent(sessionFile)) return; if (!this.isSessionFileForAgent(sessionFile)) {
return;
}
this.scheduleSessionDirty(sessionFile); this.scheduleSessionDirty(sessionFile);
}); });
} }
private scheduleSessionDirty(sessionFile: string) { private scheduleSessionDirty(sessionFile: string) {
this.sessionPendingFiles.add(sessionFile); this.sessionPendingFiles.add(sessionFile);
if (this.sessionWatchTimer) return; if (this.sessionWatchTimer) {
return;
}
this.sessionWatchTimer = setTimeout(() => { this.sessionWatchTimer = setTimeout(() => {
this.sessionWatchTimer = null; this.sessionWatchTimer = null;
void this.processSessionDeltaBatch().catch((err) => { void this.processSessionDeltaBatch().catch((err) => {
@ -771,13 +870,17 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async processSessionDeltaBatch(): Promise<void> { private async processSessionDeltaBatch(): Promise<void> {
if (this.sessionPendingFiles.size === 0) return; if (this.sessionPendingFiles.size === 0) {
return;
}
const pending = Array.from(this.sessionPendingFiles); const pending = Array.from(this.sessionPendingFiles);
this.sessionPendingFiles.clear(); this.sessionPendingFiles.clear();
let shouldSync = false; let shouldSync = false;
for (const sessionFile of pending) { for (const sessionFile of pending) {
const delta = await this.updateSessionDelta(sessionFile); const delta = await this.updateSessionDelta(sessionFile);
if (!delta) continue; if (!delta) {
continue;
}
const bytesThreshold = delta.deltaBytes; const bytesThreshold = delta.deltaBytes;
const messagesThreshold = delta.deltaMessages; const messagesThreshold = delta.deltaMessages;
const bytesHit = const bytesHit =
@ -786,7 +889,9 @@ export class MemoryIndexManager implements MemorySearchManager {
messagesThreshold <= 0 messagesThreshold <= 0
? delta.pendingMessages > 0 ? delta.pendingMessages > 0
: delta.pendingMessages >= messagesThreshold; : delta.pendingMessages >= messagesThreshold;
if (!bytesHit && !messagesHit) continue; if (!bytesHit && !messagesHit) {
continue;
}
this.sessionsDirtyFiles.add(sessionFile); this.sessionsDirtyFiles.add(sessionFile);
this.sessionsDirty = true; this.sessionsDirty = true;
delta.pendingBytes = delta.pendingBytes =
@ -809,7 +914,9 @@ export class MemoryIndexManager implements MemorySearchManager {
pendingMessages: number; pendingMessages: number;
} | null> { } | null> {
const thresholds = this.settings.sync.sessions; const thresholds = this.settings.sync.sessions;
if (!thresholds) return null; if (!thresholds) {
return null;
}
let stat: { size: number }; let stat: { size: number };
try { try {
stat = await fs.stat(sessionFile); stat = await fs.stat(sessionFile);
@ -860,7 +967,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async countNewlines(absPath: string, start: number, end: number): Promise<number> { private async countNewlines(absPath: string, start: number, end: number): Promise<number> {
if (end <= start) return 0; if (end <= start) {
return 0;
}
const handle = await fs.open(absPath, "r"); const handle = await fs.open(absPath, "r");
try { try {
let offset = start; let offset = start;
@ -869,9 +978,13 @@ export class MemoryIndexManager implements MemorySearchManager {
while (offset < end) { while (offset < end) {
const toRead = Math.min(buffer.length, end - offset); const toRead = Math.min(buffer.length, end - offset);
const { bytesRead } = await handle.read(buffer, 0, toRead, offset); const { bytesRead } = await handle.read(buffer, 0, toRead, offset);
if (bytesRead <= 0) break; if (bytesRead <= 0) {
break;
}
for (let i = 0; i < bytesRead; i += 1) { for (let i = 0; i < bytesRead; i += 1) {
if (buffer[i] === 10) count += 1; if (buffer[i] === 10) {
count += 1;
}
} }
offset += bytesRead; offset += bytesRead;
} }
@ -883,14 +996,18 @@ export class MemoryIndexManager implements MemorySearchManager {
private resetSessionDelta(absPath: string, size: number): void { private resetSessionDelta(absPath: string, size: number): void {
const state = this.sessionDeltas.get(absPath); const state = this.sessionDeltas.get(absPath);
if (!state) return; if (!state) {
return;
}
state.lastSize = size; state.lastSize = size;
state.pendingBytes = 0; state.pendingBytes = 0;
state.pendingMessages = 0; state.pendingMessages = 0;
} }
private isSessionFileForAgent(sessionFile: string): boolean { private isSessionFileForAgent(sessionFile: string): boolean {
if (!sessionFile) return false; if (!sessionFile) {
return false;
}
const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId); const sessionsDir = resolveSessionTranscriptsDirForAgent(this.agentId);
const resolvedFile = path.resolve(sessionFile); const resolvedFile = path.resolve(sessionFile);
const resolvedDir = path.resolve(sessionsDir); const resolvedDir = path.resolve(sessionsDir);
@ -899,7 +1016,9 @@ export class MemoryIndexManager implements MemorySearchManager {
private ensureIntervalSync() { private ensureIntervalSync() {
const minutes = this.settings.sync.intervalMinutes; const minutes = this.settings.sync.intervalMinutes;
if (!minutes || minutes <= 0 || this.intervalTimer) return; if (!minutes || minutes <= 0 || this.intervalTimer) {
return;
}
const ms = minutes * 60 * 1000; const ms = minutes * 60 * 1000;
this.intervalTimer = setInterval(() => { this.intervalTimer = setInterval(() => {
void this.sync({ reason: "interval" }).catch((err) => { void this.sync({ reason: "interval" }).catch((err) => {
@ -909,8 +1028,12 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private scheduleWatchSync() { private scheduleWatchSync() {
if (!this.sources.has("memory") || !this.settings.sync.watch) return; if (!this.sources.has("memory") || !this.settings.sync.watch) {
if (this.watchTimer) clearTimeout(this.watchTimer); return;
}
if (this.watchTimer) {
clearTimeout(this.watchTimer);
}
this.watchTimer = setTimeout(() => { this.watchTimer = setTimeout(() => {
this.watchTimer = null; this.watchTimer = null;
void this.sync({ reason: "watch" }).catch((err) => { void this.sync({ reason: "watch" }).catch((err) => {
@ -923,11 +1046,19 @@ export class MemoryIndexManager implements MemorySearchManager {
params?: { reason?: string; force?: boolean }, params?: { reason?: string; force?: boolean },
needsFullReindex = false, needsFullReindex = false,
) { ) {
if (!this.sources.has("sessions")) return false; if (!this.sources.has("sessions")) {
if (params?.force) return true; return false;
}
if (params?.force) {
return true;
}
const reason = params?.reason; const reason = params?.reason;
if (reason === "session-start" || reason === "watch") return false; if (reason === "session-start" || reason === "watch") {
if (needsFullReindex) return true; return false;
}
if (needsFullReindex) {
return true;
}
return this.sessionsDirty && this.sessionsDirtyFiles.size > 0; return this.sessionsDirty && this.sessionsDirtyFiles.size > 0;
} }
@ -935,7 +1066,7 @@ export class MemoryIndexManager implements MemorySearchManager {
needsFullReindex: boolean; needsFullReindex: boolean;
progress?: MemorySyncProgressState; progress?: MemorySyncProgressState;
}) { }) {
const files = await listMemoryFiles(this.workspaceDir); const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
const fileEntries = await Promise.all( const fileEntries = await Promise.all(
files.map(async (file) => buildFileEntry(file, this.workspaceDir)), files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
); );
@ -984,7 +1115,9 @@ export class MemoryIndexManager implements MemorySearchManager {
.prepare(`SELECT path FROM files WHERE source = ?`) .prepare(`SELECT path FROM files WHERE source = ?`)
.all("memory") as Array<{ path: string }>; .all("memory") as Array<{ path: string }>;
for (const stale of staleRows) { for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue; if (activePaths.has(stale.path)) {
continue;
}
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory"); this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
try { try {
this.db this.db
@ -1079,7 +1212,9 @@ export class MemoryIndexManager implements MemorySearchManager {
.prepare(`SELECT path FROM files WHERE source = ?`) .prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>; .all("sessions") as Array<{ path: string }>;
for (const stale of staleRows) { for (const stale of staleRows) {
if (activePaths.has(stale.path)) continue; if (activePaths.has(stale.path)) {
continue;
}
this.db this.db
.prepare(`DELETE FROM files WHERE path = ? AND source = ?`) .prepare(`DELETE FROM files WHERE path = ? AND source = ?`)
.run(stale.path, "sessions"); .run(stale.path, "sessions");
@ -1111,7 +1246,9 @@ export class MemoryIndexManager implements MemorySearchManager {
total: 0, total: 0,
label: undefined, label: undefined,
report: (update) => { report: (update) => {
if (update.label) state.label = update.label; if (update.label) {
state.label = update.label;
}
const label = const label =
update.total > 0 && state.label update.total > 0 && state.label
? `${state.label} ${update.completed}/${update.total}` ? `${state.label} ${update.completed}/${update.total}`
@ -1222,8 +1359,12 @@ export class MemoryIndexManager implements MemorySearchManager {
private async activateFallbackProvider(reason: string): Promise<boolean> { private async activateFallbackProvider(reason: string): Promise<boolean> {
const fallback = this.settings.fallback; const fallback = this.settings.fallback;
if (!fallback || fallback === "none" || fallback === this.provider.id) return false; if (!fallback || fallback === "none" || fallback === this.provider.id) {
if (this.fallbackFrom) return false; return false;
}
if (this.fallbackFrom) {
return false;
}
const fallbackFrom = this.provider.id as "openai" | "gemini" | "local"; const fallbackFrom = this.provider.id as "openai" | "gemini" | "local";
const fallbackModel = const fallbackModel =
@ -1375,7 +1516,9 @@ export class MemoryIndexManager implements MemorySearchManager {
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY) as
| { value: string } | { value: string }
| undefined; | undefined;
if (!row?.value) return null; if (!row?.value) {
return null;
}
try { try {
return JSON.parse(row.value) as MemoryIndexMeta; return JSON.parse(row.value) as MemoryIndexMeta;
} catch { } catch {
@ -1422,16 +1565,26 @@ export class MemoryIndexManager implements MemorySearchManager {
const normalized = this.normalizeSessionText(content); const normalized = this.normalizeSessionText(content);
return normalized ? normalized : null; return normalized ? normalized : null;
} }
if (!Array.isArray(content)) return null; if (!Array.isArray(content)) {
return null;
}
const parts: string[] = []; const parts: string[] = [];
for (const block of content) { for (const block of content) {
if (!block || typeof block !== "object") continue; if (!block || typeof block !== "object") {
continue;
}
const record = block as { type?: unknown; text?: unknown }; const record = block as { type?: unknown; text?: unknown };
if (record.type !== "text" || typeof record.text !== "string") continue; if (record.type !== "text" || typeof record.text !== "string") {
continue;
}
const normalized = this.normalizeSessionText(record.text); const normalized = this.normalizeSessionText(record.text);
if (normalized) parts.push(normalized); if (normalized) {
parts.push(normalized);
}
}
if (parts.length === 0) {
return null;
} }
if (parts.length === 0) return null;
return parts.join(" "); return parts.join(" ");
} }
@ -1442,7 +1595,9 @@ export class MemoryIndexManager implements MemorySearchManager {
const lines = raw.split("\n"); const lines = raw.split("\n");
const collected: string[] = []; const collected: string[] = [];
for (const line of lines) { for (const line of lines) {
if (!line.trim()) continue; if (!line.trim()) {
continue;
}
let record: unknown; let record: unknown;
try { try {
record = JSON.parse(line); record = JSON.parse(line);
@ -1459,10 +1614,16 @@ export class MemoryIndexManager implements MemorySearchManager {
const message = (record as { message?: unknown }).message as const message = (record as { message?: unknown }).message as
| { role?: unknown; content?: unknown } | { role?: unknown; content?: unknown }
| undefined; | undefined;
if (!message || typeof message.role !== "string") continue; if (!message || typeof message.role !== "string") {
if (message.role !== "user" && message.role !== "assistant") continue; continue;
}
if (message.role !== "user" && message.role !== "assistant") {
continue;
}
const text = this.extractSessionText(message.content); const text = this.extractSessionText(message.content);
if (!text) continue; if (!text) {
continue;
}
const label = message.role === "user" ? "User" : "Assistant"; const label = message.role === "user" ? "User" : "Assistant";
collected.push(`${label}: ${text}`); collected.push(`${label}: ${text}`);
} }
@ -1482,7 +1643,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private estimateEmbeddingTokens(text: string): number { private estimateEmbeddingTokens(text: string): number {
if (!text) return 0; if (!text) {
return 0;
}
return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN); return Math.ceil(text.length / EMBEDDING_APPROX_CHARS_PER_TOKEN);
} }
@ -1515,17 +1678,27 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private loadEmbeddingCache(hashes: string[]): Map<string, number[]> { private loadEmbeddingCache(hashes: string[]): Map<string, number[]> {
if (!this.cache.enabled) return new Map(); if (!this.cache.enabled) {
if (hashes.length === 0) return new Map(); return new Map();
}
if (hashes.length === 0) {
return new Map();
}
const unique: string[] = []; const unique: string[] = [];
const seen = new Set<string>(); const seen = new Set<string>();
for (const hash of hashes) { for (const hash of hashes) {
if (!hash) continue; if (!hash) {
if (seen.has(hash)) continue; continue;
}
if (seen.has(hash)) {
continue;
}
seen.add(hash); seen.add(hash);
unique.push(hash); unique.push(hash);
} }
if (unique.length === 0) return new Map(); if (unique.length === 0) {
return new Map();
}
const out = new Map<string, number[]>(); const out = new Map<string, number[]>();
const baseParams = [this.provider.id, this.provider.model, this.providerKey]; const baseParams = [this.provider.id, this.provider.model, this.providerKey];
@ -1547,8 +1720,12 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private upsertEmbeddingCache(entries: Array<{ hash: string; embedding: number[] }>): void { private upsertEmbeddingCache(entries: Array<{ hash: string; embedding: number[] }>): void {
if (!this.cache.enabled) return; if (!this.cache.enabled) {
if (entries.length === 0) return; return;
}
if (entries.length === 0) {
return;
}
const now = Date.now(); const now = Date.now();
const stmt = this.db.prepare( const stmt = this.db.prepare(
`INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` + `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at)\n` +
@ -1573,14 +1750,20 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private pruneEmbeddingCacheIfNeeded(): void { private pruneEmbeddingCacheIfNeeded(): void {
if (!this.cache.enabled) return; if (!this.cache.enabled) {
return;
}
const max = this.cache.maxEntries; const max = this.cache.maxEntries;
if (!max || max <= 0) return; if (!max || max <= 0) {
return;
}
const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as const row = this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as
| { c: number } | { c: number }
| undefined; | undefined;
const count = row?.c ?? 0; const count = row?.c ?? 0;
if (count <= max) return; if (count <= max) {
return;
}
const excess = count - max; const excess = count - max;
this.db this.db
.prepare( .prepare(
@ -1595,7 +1778,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> { private async embedChunksInBatches(chunks: MemoryChunk[]): Promise<number[][]> {
if (chunks.length === 0) return []; if (chunks.length === 0) {
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@ -1610,7 +1795,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
} }
if (missing.length === 0) return embeddings; if (missing.length === 0) {
return embeddings;
}
const missingChunks = missing.map((m) => m.chunk); const missingChunks = missing.map((m) => m.chunk);
const batches = this.buildEmbeddingBatches(missingChunks); const batches = this.buildEmbeddingBatches(missingChunks);
@ -1636,7 +1823,7 @@ export class MemoryIndexManager implements MemorySearchManager {
if (this.provider.id === "openai" && this.openAi) { if (this.provider.id === "openai" && this.openAi) {
const entries = Object.entries(this.openAi.headers) const entries = Object.entries(this.openAi.headers)
.filter(([key]) => key.toLowerCase() !== "authorization") .filter(([key]) => key.toLowerCase() !== "authorization")
.sort(([a], [b]) => a.localeCompare(b)) .toSorted(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => [key, value]); .map(([key, value]) => [key, value]);
return hashText( return hashText(
JSON.stringify({ JSON.stringify({
@ -1653,7 +1840,7 @@ export class MemoryIndexManager implements MemorySearchManager {
const lower = key.toLowerCase(); const lower = key.toLowerCase();
return lower !== "authorization" && lower !== "x-goog-api-key"; return lower !== "authorization" && lower !== "x-goog-api-key";
}) })
.sort(([a], [b]) => a.localeCompare(b)) .toSorted(([a], [b]) => a.localeCompare(b))
.map(([key, value]) => [key, value]); .map(([key, value]) => [key, value]);
return hashText( return hashText(
JSON.stringify({ JSON.stringify({
@ -1690,7 +1877,9 @@ export class MemoryIndexManager implements MemorySearchManager {
if (!openAi) { if (!openAi) {
return this.embedChunksInBatches(chunks); return this.embedChunksInBatches(chunks);
} }
if (chunks.length === 0) return []; if (chunks.length === 0) {
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@ -1705,7 +1894,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
} }
if (missing.length === 0) return embeddings; if (missing.length === 0) {
return embeddings;
}
const requests: OpenAiBatchRequest[] = []; const requests: OpenAiBatchRequest[] = [];
const mapping = new Map<string, { index: number; hash: string }>(); const mapping = new Map<string, { index: number; hash: string }>();
@ -1740,13 +1931,17 @@ export class MemoryIndexManager implements MemorySearchManager {
}), }),
fallback: async () => await this.embedChunksInBatches(chunks), fallback: async () => await this.embedChunksInBatches(chunks),
}); });
if (Array.isArray(batchResult)) return batchResult; if (Array.isArray(batchResult)) {
return batchResult;
}
const byCustomId = batchResult; const byCustomId = batchResult;
const toCache: Array<{ hash: string; embedding: number[] }> = []; const toCache: Array<{ hash: string; embedding: number[] }> = [];
for (const [customId, embedding] of byCustomId.entries()) { for (const [customId, embedding] of byCustomId.entries()) {
const mapped = mapping.get(customId); const mapped = mapping.get(customId);
if (!mapped) continue; if (!mapped) {
continue;
}
embeddings[mapped.index] = embedding; embeddings[mapped.index] = embedding;
toCache.push({ hash: mapped.hash, embedding }); toCache.push({ hash: mapped.hash, embedding });
} }
@ -1763,7 +1958,9 @@ export class MemoryIndexManager implements MemorySearchManager {
if (!gemini) { if (!gemini) {
return this.embedChunksInBatches(chunks); return this.embedChunksInBatches(chunks);
} }
if (chunks.length === 0) return []; if (chunks.length === 0) {
return [];
}
const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash)); const cached = this.loadEmbeddingCache(chunks.map((chunk) => chunk.hash));
const embeddings: number[][] = Array.from({ length: chunks.length }, () => []); const embeddings: number[][] = Array.from({ length: chunks.length }, () => []);
const missing: Array<{ index: number; chunk: MemoryChunk }> = []; const missing: Array<{ index: number; chunk: MemoryChunk }> = [];
@ -1778,7 +1975,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
} }
if (missing.length === 0) return embeddings; if (missing.length === 0) {
return embeddings;
}
const requests: GeminiBatchRequest[] = []; const requests: GeminiBatchRequest[] = [];
const mapping = new Map<string, { index: number; hash: string }>(); const mapping = new Map<string, { index: number; hash: string }>();
@ -1810,13 +2009,17 @@ export class MemoryIndexManager implements MemorySearchManager {
}), }),
fallback: async () => await this.embedChunksInBatches(chunks), fallback: async () => await this.embedChunksInBatches(chunks),
}); });
if (Array.isArray(batchResult)) return batchResult; if (Array.isArray(batchResult)) {
return batchResult;
}
const byCustomId = batchResult; const byCustomId = batchResult;
const toCache: Array<{ hash: string; embedding: number[] }> = []; const toCache: Array<{ hash: string; embedding: number[] }> = [];
for (const [customId, embedding] of byCustomId.entries()) { for (const [customId, embedding] of byCustomId.entries()) {
const mapped = mapping.get(customId); const mapped = mapping.get(customId);
if (!mapped) continue; if (!mapped) {
continue;
}
embeddings[mapped.index] = embedding; embeddings[mapped.index] = embedding;
toCache.push({ hash: mapped.hash, embedding }); toCache.push({ hash: mapped.hash, embedding });
} }
@ -1825,7 +2028,9 @@ export class MemoryIndexManager implements MemorySearchManager {
} }
private async embedBatchWithRetry(texts: string[]): Promise<number[][]> { private async embedBatchWithRetry(texts: string[]): Promise<number[][]> {
if (texts.length === 0) return []; if (texts.length === 0) {
return [];
}
let attempt = 0; let attempt = 0;
let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS; let delayMs = EMBEDDING_RETRY_BASE_DELAY_MS;
while (true) { while (true) {
@ -1887,7 +2092,9 @@ export class MemoryIndexManager implements MemorySearchManager {
timeoutMs: number, timeoutMs: number,
message: string, message: string,
): Promise<T> { ): Promise<T> {
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return await promise; if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
return await promise;
}
let timer: NodeJS.Timeout | null = null; let timer: NodeJS.Timeout | null = null;
const timeoutPromise = new Promise<never>((_, reject) => { const timeoutPromise = new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error(message)), timeoutMs); timer = setTimeout(() => reject(new Error(message)), timeoutMs);
@ -1895,12 +2102,16 @@ export class MemoryIndexManager implements MemorySearchManager {
try { try {
return (await Promise.race([promise, timeoutPromise])) as T; return (await Promise.race([promise, timeoutPromise])) as T;
} finally { } finally {
if (timer) clearTimeout(timer); if (timer) {
clearTimeout(timer);
}
} }
} }
private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> { private async runWithConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]> {
if (tasks.length === 0) return []; if (tasks.length === 0) {
return [];
}
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length)); const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
const results: T[] = Array.from({ length: tasks.length }); const results: T[] = Array.from({ length: tasks.length });
let next = 0; let next = 0;
@ -1908,10 +2119,14 @@ export class MemoryIndexManager implements MemorySearchManager {
const workers = Array.from({ length: resolvedLimit }, async () => { const workers = Array.from({ length: resolvedLimit }, async () => {
while (true) { while (true) {
if (firstError) return; if (firstError) {
return;
}
const index = next; const index = next;
next += 1; next += 1;
if (index >= tasks.length) return; if (index >= tasks.length) {
return;
}
try { try {
results[index] = await tasks[index](); results[index] = await tasks[index]();
} catch (err) { } catch (err) {
@ -1922,7 +2137,9 @@ export class MemoryIndexManager implements MemorySearchManager {
}); });
await Promise.allSettled(workers); await Promise.allSettled(workers);
if (firstError) throw firstError; if (firstError) {
throw firstError;
}
return results; return results;
} }

1
src/memory/types.ts
View File

@ -31,6 +31,7 @@ export type MemoryProviderStatus = {
dirty?: boolean; dirty?: boolean;
workspaceDir?: string; workspaceDir?: string;
dbPath?: string; dbPath?: string;
extraPaths?: string[];
sources?: MemorySource[]; sources?: MemorySource[];
sourceCounts?: Array<{ source: MemorySource; files: number; chunks: number }>; sourceCounts?: Array<{ source: MemorySource; files: number; chunks: number }>;
cache?: { enabled: boolean; entries?: number; maxEntries?: number }; cache?: { enabled: boolean; entries?: number; maxEntries?: number };