/** * CLI Commands for Memory Management */ import type { Command } from "commander"; import { readFileSync } from "node:fs"; import { loadLanceDB, type MemoryEntry, type MemoryStore } from "./src/store.js"; import type { MemoryRetriever } from "./src/retriever.js"; import type { MemoryScopeManager } from "./src/scopes.js"; import type { MemoryMigrator } from "./src/migrate.js"; // ============================================================================ // Types // ============================================================================ interface CLIContext { store: MemoryStore; retriever: MemoryRetriever; scopeManager: MemoryScopeManager; migrator: MemoryMigrator; embedder?: import("./src/embedder.js").Embedder; } // ============================================================================ // Utility Functions // ============================================================================ function getPluginVersion(): string { try { const pkgUrl = new URL("./package.json", import.meta.url); const pkg = JSON.parse(readFileSync(pkgUrl, "utf8")) as { version?: string }; return pkg.version || "unknown"; } catch { return "unknown"; } } function clampInt(value: number, min: number, max: number): number { const n = Number.isFinite(value) ? value : min; return Math.max(min, Math.min(max, Math.trunc(n))); } function formatMemory(memory: any, index?: number): string { const prefix = index !== undefined ? `${index + 1}. ` : ""; const id = memory?.id ? String(memory.id) : "unknown"; const date = new Date(memory.timestamp || memory.createdAt || Date.now()).toISOString().split('T')[0]; const fullText = String(memory.text || ""); const text = fullText.slice(0, 100) + (fullText.length > 100 ? "..." : ""); return `${prefix}[${id}] [${memory.category}:${memory.scope}] ${text} (${date})`; } function formatJson(obj: any): string { return JSON.stringify(obj, null, 2); } // ============================================================================ // CLI Command Implementations // ============================================================================ export function registerMemoryCLI(program: Command, context: CLIContext): void { const memory = program .command("memory-pro") .description("Enhanced memory management commands (LanceDB Pro)"); // Version memory .command("version") .description("Print plugin version") .action(() => { console.log(getPluginVersion()); }); // List memories memory .command("list") .description("List memories with optional filtering") .option("--scope ", "Filter by scope") .option("--category ", "Filter by category") .option("--limit ", "Maximum number of results", "20") .option("--offset ", "Number of results to skip", "0") .option("--json", "Output as JSON") .action(async (options) => { try { const limit = parseInt(options.limit) || 20; const offset = parseInt(options.offset) || 0; let scopeFilter: string[] | undefined; if (options.scope) { scopeFilter = [options.scope]; } const memories = await context.store.list( scopeFilter, options.category, limit, offset ); if (options.json) { console.log(formatJson(memories)); } else { if (memories.length === 0) { console.log("No memories found."); } else { console.log(`Found ${memories.length} memories:\n`); memories.forEach((memory, i) => { console.log(formatMemory(memory, offset + i)); }); } } } catch (error) { console.error("Failed to list memories:", error); process.exit(1); } }); // Search memories memory .command("search ") .description("Search memories using hybrid retrieval") .option("--scope ", "Search within specific scope") .option("--category ", "Filter by category") .option("--limit ", "Maximum number of results", "10") .option("--json", "Output as JSON") .action(async (query, options) => { try { const limit = parseInt(options.limit) || 10; let scopeFilter: string[] | undefined; if (options.scope) { scopeFilter = [options.scope]; } const results = await context.retriever.retrieve({ query, limit, scopeFilter, category: options.category, }); if (options.json) { console.log(formatJson(results)); } else { if (results.length === 0) { console.log("No relevant memories found."); } else { console.log(`Found ${results.length} memories:\n`); results.forEach((result, i) => { const sources = []; if (result.sources.vector) sources.push("vector"); if (result.sources.bm25) sources.push("BM25"); if (result.sources.reranked) sources.push("reranked"); console.log( `${i + 1}. [${result.entry.id}] [${result.entry.category}:${result.entry.scope}] ${result.entry.text} ` + `(${(result.score * 100).toFixed(0)}%, ${sources.join('+')})` ); }); } } } catch (error) { console.error("Search failed:", error); process.exit(1); } }); // Memory statistics memory .command("stats") .description("Show memory statistics") .option("--scope ", "Stats for specific scope") .option("--json", "Output as JSON") .action(async (options) => { try { let scopeFilter: string[] | undefined; if (options.scope) { scopeFilter = [options.scope]; } const stats = await context.store.stats(scopeFilter); const scopeStats = context.scopeManager.getStats(); const retrievalConfig = context.retriever.getConfig(); const summary = { memory: stats, scopes: scopeStats, retrieval: { mode: retrievalConfig.mode, hasFtsSupport: context.store.hasFtsSupport, }, }; if (options.json) { console.log(formatJson(summary)); } else { console.log(`Memory Statistics:`); console.log(`• Total memories: ${stats.totalCount}`); console.log(`• Available scopes: ${scopeStats.totalScopes}`); console.log(`• Retrieval mode: ${retrievalConfig.mode}`); console.log(`• FTS support: ${context.store.hasFtsSupport ? 'Yes' : 'No'}`); console.log(); console.log("Memories by scope:"); Object.entries(stats.scopeCounts).forEach(([scope, count]) => { console.log(` • ${scope}: ${count}`); }); console.log(); console.log("Memories by category:"); Object.entries(stats.categoryCounts).forEach(([category, count]) => { console.log(` • ${category}: ${count}`); }); } } catch (error) { console.error("Failed to get statistics:", error); process.exit(1); } }); // Delete memory memory .command("delete ") .description("Delete a specific memory by ID") .option("--scope ", "Scope to delete from (for access control)") .action(async (id, options) => { try { let scopeFilter: string[] | undefined; if (options.scope) { scopeFilter = [options.scope]; } const deleted = await context.store.delete(id, scopeFilter); if (deleted) { console.log(`Memory ${id} deleted successfully.`); } else { console.log(`Memory ${id} not found or access denied.`); process.exit(1); } } catch (error) { console.error("Failed to delete memory:", error); process.exit(1); } }); // Bulk delete memory .command("delete-bulk") .description("Bulk delete memories with filters") .option("--scope ", "Scopes to delete from (required)") .option("--before ", "Delete memories before this date (YYYY-MM-DD)") .option("--dry-run", "Show what would be deleted without actually deleting") .action(async (options) => { try { if (!options.scope || options.scope.length === 0) { console.error("At least one scope must be specified for safety."); process.exit(1); } let beforeTimestamp: number | undefined; if (options.before) { const date = new Date(options.before); if (isNaN(date.getTime())) { console.error("Invalid date format. Use YYYY-MM-DD."); process.exit(1); } beforeTimestamp = date.getTime(); } if (options.dryRun) { console.log("DRY RUN - No memories will be deleted"); console.log(`Filters: scopes=${options.scope.join(',')}, before=${options.before || 'none'}`); // Show what would be deleted const stats = await context.store.stats(options.scope); console.log(`Would delete from ${stats.totalCount} memories in matching scopes.`); } else { const deletedCount = await context.store.bulkDelete(options.scope, beforeTimestamp); console.log(`Deleted ${deletedCount} memories.`); } } catch (error) { console.error("Bulk delete failed:", error); process.exit(1); } }); // Export memories memory .command("export") .description("Export memories to JSON") .option("--scope ", "Export specific scope") .option("--category ", "Export specific category") .option("--output ", "Output file (default: stdout)") .action(async (options) => { try { let scopeFilter: string[] | undefined; if (options.scope) { scopeFilter = [options.scope]; } const memories = await context.store.list( scopeFilter, options.category, 1000 // Large limit for export ); const exportData = { version: "1.0", exportedAt: new Date().toISOString(), count: memories.length, filters: { scope: options.scope, category: options.category, }, memories: memories.map(m => ({ ...m, vector: undefined, // Exclude vectors to reduce size })), }; const output = formatJson(exportData); if (options.output) { const fs = await import("node:fs/promises"); await fs.writeFile(options.output, output); console.log(`Exported ${memories.length} memories to ${options.output}`); } else { console.log(output); } } catch (error) { console.error("Export failed:", error); process.exit(1); } }); // Import memories memory .command("import ") .description("Import memories from JSON file") .option("--scope ", "Import into specific scope") .option("--dry-run", "Show what would be imported without actually importing") .action(async (file, options) => { try { const fs = await import("node:fs/promises"); const content = await fs.readFile(file, "utf-8"); const data = JSON.parse(content); if (!data.memories || !Array.isArray(data.memories)) { throw new Error("Invalid import file format"); } if (options.dryRun) { console.log("DRY RUN - No memories will be imported"); console.log(`Would import ${data.memories.length} memories`); if (options.scope) { console.log(`Target scope: ${options.scope}`); } return; } console.log(`Importing ${data.memories.length} memories...`); let imported = 0; let skipped = 0; if (!context.embedder) { console.error("Import requires an embedder (not available in basic CLI mode)."); console.error("Use the plugin's memory_store tool or pass embedder to createMemoryCLI."); return; } const targetScope = options.scope || context.scopeManager.getDefaultScope(); for (const memory of data.memories) { try { const text = memory.text; if (!text || typeof text !== "string" || text.length < 2) { skipped++; continue; } // Check for duplicates const existing = await context.retriever.retrieve({ query: text, limit: 1, scopeFilter: [targetScope], }); if (existing.length > 0 && existing[0].score > 0.95) { skipped++; continue; } const vector = await context.embedder.embedPassage(text); await context.store.store({ text, vector, importance: memory.importance ?? 0.7, category: memory.category || "other", scope: targetScope, }); imported++; } catch (error) { console.warn(`Failed to import memory: ${error}`); skipped++; } } console.log(`Import completed: ${imported} imported, ${skipped} skipped`); } catch (error) { console.error("Import failed:", error); process.exit(1); } }); // Re-embed an existing LanceDB into the current target DB (A/B testing) memory .command("reembed") .description("Re-embed memories from a source LanceDB database into the current target database") .requiredOption("--source-db ", "Source LanceDB database directory") .option("--batch-size ", "Batch size for embedding calls", "32") .option("--limit ", "Limit number of rows to process (for testing)") .option("--dry-run", "Show what would be re-embedded without writing") .option("--skip-existing", "Skip entries whose id already exists in the target DB") .option("--force", "Allow using the same source-db as the target dbPath (DANGEROUS)") .action(async (options) => { try { if (!context.embedder) { console.error("Re-embed requires an embedder (not available in basic CLI mode)."); return; } const fs = await import("node:fs/promises"); const sourceDbPath = options.sourceDb as string; const batchSize = clampInt(parseInt(options.batchSize, 10) || 32, 1, 128); const limit = options.limit ? clampInt(parseInt(options.limit, 10) || 0, 1, 1000000) : undefined; const dryRun = options.dryRun === true; const skipExisting = options.skipExisting === true; const force = options.force === true; // Safety: prevent accidental in-place re-embedding let sourceReal = sourceDbPath; let targetReal = context.store.dbPath; try { sourceReal = await fs.realpath(sourceDbPath); } catch {} try { targetReal = await fs.realpath(context.store.dbPath); } catch {} if (!force && sourceReal === targetReal) { console.error("Refusing to re-embed in-place: source-db equals target dbPath. Use a new dbPath or pass --force."); process.exit(1); } const lancedb = await loadLanceDB(); const db = await lancedb.connect(sourceDbPath); const table = await db.openTable("memories"); let query = table .query() .select(["id", "text", "category", "scope", "importance", "timestamp", "metadata"]); if (limit) query = query.limit(limit); const rows = (await query.toArray()) .filter((r: any) => r && typeof r.text === "string" && r.text.trim().length > 0) .filter((r: any) => r.id && r.id !== "__schema__"); if (rows.length === 0) { console.log("No source memories found."); return; } console.log( `Re-embedding ${rows.length} memories from ${sourceDbPath} → ${context.store.dbPath} (batchSize=${batchSize})` ); if (dryRun) { console.log("DRY RUN - No memories will be written"); console.log(`First example: ${rows[0].id?.slice?.(0, 8)} ${String(rows[0].text).slice(0, 80)}`); return; } let processed = 0; let imported = 0; let skipped = 0; for (let i = 0; i < rows.length; i += batchSize) { const batch = rows.slice(i, i + batchSize); const texts = batch.map((r: any) => String(r.text)); const vectors = await context.embedder.embedBatchPassage(texts); for (let j = 0; j < batch.length; j++) { processed++; const row = batch[j]; const vector = vectors[j]; if (!vector || vector.length === 0) { skipped++; continue; } const id = String(row.id); if (skipExisting) { const exists = await context.store.hasId(id); if (exists) { skipped++; continue; } } const entry: MemoryEntry = { id, text: String(row.text), vector, category: (row.category as any) || "other", scope: (row.scope as string | undefined) || "global", importance: (row.importance != null) ? Number(row.importance) : 0.7, timestamp: (row.timestamp != null) ? Number(row.timestamp) : Date.now(), metadata: typeof row.metadata === "string" ? row.metadata : "{}", }; await context.store.importEntry(entry); imported++; } if (processed % 100 === 0 || processed === rows.length) { console.log(`Progress: ${processed}/${rows.length} processed, ${imported} imported, ${skipped} skipped`); } } console.log(`Re-embed completed: ${imported} imported, ${skipped} skipped (processed=${processed}).`); } catch (error) { console.error("Re-embed failed:", error); process.exit(1); } }); // Migration commands const migrate = memory .command("migrate") .description("Migration utilities"); migrate .command("check") .description("Check if migration is needed from legacy memory-lancedb") .option("--source ", "Specific source database path") .action(async (options) => { try { const check = await context.migrator.checkMigrationNeeded(options.source); console.log("Migration Check Results:"); console.log(`• Legacy database found: ${check.sourceFound ? 'Yes' : 'No'}`); if (check.sourceDbPath) { console.log(`• Source path: ${check.sourceDbPath}`); } if (check.entryCount !== undefined) { console.log(`• Entries to migrate: ${check.entryCount}`); } console.log(`• Migration needed: ${check.needed ? 'Yes' : 'No'}`); } catch (error) { console.error("Migration check failed:", error); process.exit(1); } }); migrate .command("run") .description("Run migration from legacy memory-lancedb") .option("--source ", "Specific source database path") .option("--default-scope ", "Default scope for migrated data", "global") .option("--dry-run", "Show what would be migrated without actually migrating") .option("--skip-existing", "Skip entries that already exist") .action(async (options) => { try { const result = await context.migrator.migrate({ sourceDbPath: options.source, defaultScope: options.defaultScope, dryRun: options.dryRun, skipExisting: options.skipExisting, }); console.log("Migration Results:"); console.log(`• Status: ${result.success ? 'Success' : 'Failed'}`); console.log(`• Migrated: ${result.migratedCount}`); console.log(`• Skipped: ${result.skippedCount}`); if (result.errors.length > 0) { console.log(`• Errors: ${result.errors.length}`); result.errors.forEach(error => console.log(` - ${error}`)); } console.log(`• Summary: ${result.summary}`); if (!result.success) { process.exit(1); } } catch (error) { console.error("Migration failed:", error); process.exit(1); } }); migrate .command("verify") .description("Verify migration results") .option("--source ", "Specific source database path") .action(async (options) => { try { const result = await context.migrator.verifyMigration(options.source); console.log("Migration Verification:"); console.log(`• Valid: ${result.valid ? 'Yes' : 'No'}`); console.log(`• Source count: ${result.sourceCount}`); console.log(`• Target count: ${result.targetCount}`); if (result.issues.length > 0) { console.log("• Issues:"); result.issues.forEach(issue => console.log(` - ${issue}`)); } if (!result.valid) { process.exit(1); } } catch (error) { console.error("Verification failed:", error); process.exit(1); } }); } // ============================================================================ // Factory Function // ============================================================================ export function createMemoryCLI(context: CLIContext) { return ({ program }: { program: Command }) => registerMemoryCLI(program, context); }