From d11f19e7038f44dd7d8090da082b64fe2f5464e6 Mon Sep 17 00:00:00 2001
From: root <root@dflike.hookstrapmx>
Date: Tue, 10 Mar 2026 13:12:38 +0000
Subject: [PATCH] docs: add LLM analytics implementation plan

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 docs/plans/2026-03-10-llm-analytics-plan.md | 1324 +++++++++++++++++++
 1 file changed, 1324 insertions(+)
 create mode 100644 docs/plans/2026-03-10-llm-analytics-plan.md

diff --git a/docs/plans/2026-03-10-llm-analytics-plan.md b/docs/plans/2026-03-10-llm-analytics-plan.md
new file mode 100644
index 0000000..fdda911
--- /dev/null
+++ b/docs/plans/2026-03-10-llm-analytics-plan.md
@@ -0,0 +1,1324 @@
+# LLM Analytics Tracking Implementation Plan
+
+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
+
+**Goal:** Track every LLM call by type with cost calculations, retry/failure rates, and surface analytics in the admin panel.
+
+**Architecture:** Instrument `llmService.generate()` to record each call's tokens, cost, retries, and failure status to a new SQLite table. A new `llmStatsService` handles recording and aggregation. The admin panel widens to 680px when active, with a new right column showing per-type analytics.
+
+**Tech Stack:** SQLite (better-sqlite3), Socket.io, TypeScript, HTML/CSS
+
+---
+
+### Task 1: Add LLM Stats Types to Shared
+
+**Files:**
+- Modify: `shared/src/types.ts`
+
+**Step 1: Add LLM stats types and socket events**
+
+Add the following types and socket event declarations to `shared/src/types.ts`:
+
+After the `LogEntry` interface (around line 310), add:
+
+```typescript
+export interface LlmCallTypeStats {
+  templateName: string;
+  label: string;
+  count: number;
+  totalCost: number;
+  minCost: number;
+  maxCost: number;
+  avgCost: number;
+  retryCount: number;
+  failCount: number;
+  retryPct: number;
+  failPct: number;
+}
+
+export interface LlmStatsData {
+  types: LlmCallTypeStats[];
+  totalCount: number;
+  totalCost: number;
+  totalRetries: number;
+  totalFailures: number;
+  trackingEnabled: boolean;
+}
+```
+
+Add to the `ServerEvents` interface (before the closing `}`):
+
+```typescript
+  'admin-llm-stats-result': (data: LlmStatsData) => void;
+  'admin-llm-tracking-toggled': (data: { enabled: boolean }) => void;
+```
+
+Add to the `ClientEvents` interface (before the closing `}`):
+
+```typescript
+  'admin-llm-stats': () => void;
+  'admin-toggle-llm-tracking': (data: { enabled: boolean }) => void;
+```
+
+**Step 2: Rebuild shared types**
+
+Run: `npx -w shared tsc`
+Expected: Clean build, no errors.
+
+**Step 3: Commit**
+
+```bash
+git add shared/src/types.ts
+git commit -m "feat: add LLM analytics types and socket events to shared"
+```
+
+---
+
+### Task 2: Add Model Pricing Config
+
+**Files:**
+- Modify: `server/src/config/llmConfig.ts`
+
+**Step 1: Write test for pricing config**
+
+Create `server/src/config/__tests__/llmConfig.test.ts`:
+
+```typescript
+import { describe, it, expect } from 'vitest';
+import { MODEL_PRICING, calculateCost } from '../llmConfig.js';
+
+describe('MODEL_PRICING', () => {
+  it('has free pricing for trinity models', () => {
+    const pricing = MODEL_PRICING['recursal/trinity-large'];
+    expect(pricing).toEqual({ inputPerToken: 0, outputPerToken: 0 });
+  });
+
+  it('has correct pricing for gpt-oss-120b', () => {
+    const pricing = MODEL_PRICING['gpt-oss-120b'];
+    expect(pricing.inputPerToken).toBeCloseTo(0.000000039, 12);
+    expect(pricing.outputPerToken).toBeCloseTo(0.00000019, 12);
+  });
+});
+
+describe('calculateCost', () => {
+  it('returns 0 for free model', () => {
+    expect(calculateCost('recursal/trinity-large', 1000, 500)).toBe(0);
+  });
+
+  it('calculates cost for paid model', () => {
+    const cost = calculateCost('gpt-oss-120b', 1000, 500);
+    // 1000 * 0.000000039 + 500 * 0.00000019 = 0.000039 + 0.000095 = 0.000134
+    expect(cost).toBeCloseTo(0.000134, 8);
+  });
+
+  it('returns 0 for unknown model', () => {
+    expect(calculateCost('unknown-model', 1000, 500)).toBe(0);
+  });
+});
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `npm -w server run test -- --run src/config/__tests__/llmConfig.test.ts`
+Expected: FAIL — `MODEL_PRICING` and `calculateCost` not exported.
+
+**Step 3: Implement pricing config**
+
+Add to the end of `server/src/config/llmConfig.ts`:
+
+```typescript
+export interface ModelPricing {
+  inputPerToken: number;
+  outputPerToken: number;
+}
+
+export const MODEL_PRICING: Record<string, ModelPricing> = {
+  'recursal/trinity-large': { inputPerToken: 0, outputPerToken: 0 },
+  'gpt-oss-120b': { inputPerToken: 0.039 / 1_000_000, outputPerToken: 0.19 / 1_000_000 },
+};
+
+export function calculateCost(model: string, inputTokens: number, outputTokens: number): number {
+  const pricing = MODEL_PRICING[model];
+  if (!pricing) return 0;
+  return inputTokens * pricing.inputPerToken + outputTokens * pricing.outputPerToken;
+}
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `npm -w server run test -- --run src/config/__tests__/llmConfig.test.ts`
+Expected: PASS
+
+**Step 5: Commit**
+
+```bash
+git add server/src/config/llmConfig.ts server/src/config/__tests__/llmConfig.test.ts
+git commit -m "feat: add model pricing config and cost calculation"
+```
+
+---
+
+### Task 3: Modify openRouterClient to Return Retry Count
+
+**Files:**
+- Modify: `server/src/llm/openRouterClient.ts`
+- Modify: `server/src/llm/__tests__/openRouterClient.test.ts`
+
+**Step 1: Write tests for retry count in result**
+
+Add to `server/src/llm/__tests__/openRouterClient.test.ts`:
+
+```typescript
+it('CompletionSuccess includes retries count of 0 on first success', async () => {
+  mockFetchOk('hello');
+  const client = createOpenRouterClient(defaultConfig());
+  const result = await client.complete(makeRequest());
+  expect(isSuccess(result)).toBe(true);
+  if (isSuccess(result)) {
+    expect(result.retries).toBe(0);
+  }
+});
+
+it('CompletionSuccess includes retry count after server errors', async () => {
+  let callCount = 0;
+  globalThis.fetch = vi.fn().mockImplementation(() => {
+    callCount++;
+    if (callCount <= 1) {
+      return Promise.resolve({ ok: false, status: 500, statusText: 'Server Error' });
+    }
+    return Promise.resolve({
+      ok: true,
+      json: () => Promise.resolve({ choices: [{ message: { content: 'ok' } }], usage: { prompt_tokens: 10, completion_tokens: 5 } }),
+    });
+  });
+  const client = createOpenRouterClient(defaultConfig());
+  const result = await client.complete(makeRequest());
+  expect(isSuccess(result)).toBe(true);
+  if (isSuccess(result)) {
+    expect(result.retries).toBe(1);
+  }
+});
+```
+
+Note: Check the existing test file for helper functions like `defaultConfig()` and `makeRequest()` — use whatever patterns are already there.
+
+**Step 2: Run tests to verify they fail**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/openRouterClient.test.ts`
+Expected: FAIL — `retries` not in `CompletionSuccess`.
+
+**Step 3: Add retries to CompletionSuccess**
+
+Modify `server/src/llm/openRouterClient.ts`:
+
+Change the `CompletionSuccess` interface (line 35-38) to:
+
+```typescript
+export interface CompletionSuccess {
+  content: string;
+  usage: TokenUsage | null;
+  retries: number;
+}
+```
+
+In the `complete` method, track retries. The `attempt` variable in the loop already tracks this. Change the success return (around line 106-112) to include `retries: attempt`:
+
+```typescript
+          return {
+            content,
+            usage: usage ? {
+              promptTokens: usage.prompt_tokens ?? 0,
+              completionTokens: usage.completion_tokens ?? 0,
+            } : null,
+            retries: attempt,
+          };
+```
+
+**Step 4: Run tests to verify they pass**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/openRouterClient.test.ts`
+Expected: PASS
+
+**Step 5: Run all tests to check nothing broke**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 6: Commit**
+
+```bash
+git add server/src/llm/openRouterClient.ts server/src/llm/__tests__/openRouterClient.test.ts
+git commit -m "feat: include retry count in openRouterClient CompletionSuccess"
+```
+
+---
+
+### Task 4: Create llmStatsService
+
+**Files:**
+- Create: `server/src/llm/llmStatsService.ts`
+- Create: `server/src/llm/__tests__/llmStatsService.test.ts`
+
+**Step 1: Write tests**
+
+Create `server/src/llm/__tests__/llmStatsService.test.ts`:
+
+```typescript
+import { describe, it, expect, afterEach } from 'vitest';
+import { createLlmStatsService } from '../llmStatsService.js';
+import { openDatabase, closeDatabase, getDatabase } from '../../persistence/database.js';
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+
+function tempDbPath(): string {
+  return path.join(os.tmpdir(), `dflike-stats-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`);
+}
+
+describe('llmStatsService', () => {
+  const dbPaths: string[] = [];
+
+  function setup(): ReturnType<typeof createLlmStatsService> {
+    const p = tempDbPath();
+    dbPaths.push(p);
+    openDatabase(p);
+    // Create the llm_call_log table manually for unit tests
+    const db = getDatabase();
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS llm_call_log (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        tick INTEGER,
+        template_name TEXT NOT NULL,
+        input_tokens INTEGER NOT NULL,
+        output_tokens INTEGER NOT NULL,
+        cost_usd REAL NOT NULL,
+        model TEXT NOT NULL,
+        retries INTEGER NOT NULL DEFAULT 0,
+        failed INTEGER NOT NULL DEFAULT 0,
+        timestamp TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_llm_call_log_template ON llm_call_log(template_name);
+    `);
+    // Set tracking enabled
+    db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES ('llm_tracking_enabled', '1')").run();
+    return createLlmStatsService();
+  }
+
+  afterEach(() => {
+    try { closeDatabase(); } catch { /* ignore */ }
+    for (const p of dbPaths) {
+      try { fs.unlinkSync(p); } catch { /* ignore */ }
+      try { fs.unlinkSync(p + '-wal'); } catch { /* ignore */ }
+      try { fs.unlinkSync(p + '-shm'); } catch { /* ignore */ }
+    }
+    dbPaths.length = 0;
+  });
+
+  it('records a successful call and retrieves stats', () => {
+    const service = setup();
+    service.record('socialNarration', 100, 50, 'gpt-oss-120b', 0, false, 42);
+
+    const stats = service.getStats();
+    expect(stats.types).toHaveLength(1);
+    expect(stats.types[0].templateName).toBe('socialNarration');
+    expect(stats.types[0].count).toBe(1);
+    expect(stats.types[0].retryCount).toBe(0);
+    expect(stats.types[0].failCount).toBe(0);
+    expect(stats.types[0].totalCost).toBeGreaterThan(0);
+    expect(stats.totalCount).toBe(1);
+  });
+
+  it('tracks retries and failures', () => {
+    const service = setup();
+    service.record('socialNarration', 100, 50, 'gpt-oss-120b', 2, false, 1);
+    service.record('socialNarration', 100, 50, 'gpt-oss-120b', 0, true, 2);
+
+    const stats = service.getStats();
+    expect(stats.types[0].retryCount).toBe(1); // 1 call had retries > 0
+    expect(stats.types[0].failCount).toBe(1);
+    expect(stats.types[0].retryPct).toBe(50);
+    expect(stats.types[0].failPct).toBe(50);
+  });
+
+  it('calculates min/max/avg cost', () => {
+    const service = setup();
+    // Free model = $0
+    service.record('backstoryAndDesires', 100, 50, 'recursal/trinity-large', 0, false, 1);
+    // Paid model
+    service.record('backstoryAndDesires', 1000, 500, 'gpt-oss-120b', 0, false, 2);
+
+    const stats = service.getStats();
+    const bs = stats.types.find(t => t.templateName === 'backstoryAndDesires')!;
+    expect(bs.count).toBe(2);
+    expect(bs.minCost).toBe(0);
+    expect(bs.maxCost).toBeGreaterThan(0);
+    expect(bs.avgCost).toBeGreaterThan(0);
+    expect(bs.avgCost).toBeLessThan(bs.maxCost);
+  });
+
+  it('respects tracking toggle', () => {
+    const service = setup();
+    service.setTrackingEnabled(false);
+    service.record('socialNarration', 100, 50, 'gpt-oss-120b', 0, false, 1);
+
+    const stats = service.getStats();
+    expect(stats.totalCount).toBe(0);
+    expect(stats.trackingEnabled).toBe(false);
+  });
+
+  it('returns friendly labels', () => {
+    const service = setup();
+    service.record('backstoryAndDesires', 100, 50, 'recursal/trinity-large', 0, false, 1);
+    service.record('socialNarration', 100, 50, 'recursal/trinity-large', 0, false, 2);
+    service.record('batchedThoughts', 100, 50, 'recursal/trinity-large', 0, false, 3);
+    service.record('desireGeneration', 100, 50, 'recursal/trinity-large', 0, false, 4);
+    service.record('invention', 100, 50, 'recursal/trinity-large', 0, false, 5);
+
+    const stats = service.getStats();
+    const labels = stats.types.map(t => t.label);
+    expect(labels).toContain('Backstory');
+    expect(labels).toContain('Social');
+    expect(labels).toContain('Thoughts');
+    expect(labels).toContain('Desires');
+    expect(labels).toContain('Invention');
+  });
+});
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/llmStatsService.test.ts`
+Expected: FAIL — module not found.
+
+**Step 3: Implement llmStatsService**
+
+Create `server/src/llm/llmStatsService.ts`:
+
+```typescript
+import { getDatabase } from '../persistence/database.js';
+import { calculateCost } from '../config/llmConfig.js';
+import type { LlmStatsData, LlmCallTypeStats } from '@dflike/shared';
+
+const TEMPLATE_LABELS: Record<string, string> = {
+  backstoryAndDesires: 'Backstory',
+  backstory: 'Backstory (old)',
+  socialNarration: 'Social',
+  batchedThoughts: 'Thoughts',
+  desireGeneration: 'Desires',
+  invention: 'Invention',
+};
+
+export interface LlmStatsService {
+  record(templateName: string, inputTokens: number, outputTokens: number, model: string, retries: number, failed: boolean, tick: number): void;
+  getStats(): LlmStatsData;
+  isTrackingEnabled(): boolean;
+  setTrackingEnabled(enabled: boolean): void;
+}
+
+export function createLlmStatsService(): LlmStatsService {
+  let trackingEnabled: boolean | null = null;
+
+  function loadTrackingEnabled(): boolean {
+    if (trackingEnabled !== null) return trackingEnabled;
+    try {
+      const db = getDatabase();
+      const row = db.prepare("SELECT value FROM metadata WHERE key = 'llm_tracking_enabled'").get() as { value: string } | undefined;
+      trackingEnabled = row?.value !== '0';
+    } catch {
+      trackingEnabled = true;
+    }
+    return trackingEnabled;
+  }
+
+  return {
+    record(templateName: string, inputTokens: number, outputTokens: number, model: string, retries: number, failed: boolean, tick: number): void {
+      if (!loadTrackingEnabled()) return;
+      const cost = calculateCost(model, inputTokens, outputTokens);
+      try {
+        const db = getDatabase();
+        db.prepare(`
+          INSERT INTO llm_call_log (tick, template_name, input_tokens, output_tokens, cost_usd, model, retries, failed, timestamp)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+        `).run(tick, templateName, inputTokens, outputTokens, cost, model, retries, failed ? 1 : 0, new Date().toISOString());
+      } catch {
+        // Silently ignore DB errors to not break gameplay
+      }
+    },
+
+    getStats(): LlmStatsData {
+      const enabled = loadTrackingEnabled();
+      try {
+        const db = getDatabase();
+        const rows = db.prepare(`
+          SELECT
+            template_name,
+            COUNT(*) as count,
+            SUM(cost_usd) as total_cost,
+            MIN(cost_usd) as min_cost,
+            MAX(cost_usd) as max_cost,
+            AVG(cost_usd) as avg_cost,
+            SUM(CASE WHEN retries > 0 THEN 1 ELSE 0 END) as retry_count,
+            SUM(failed) as fail_count
+          FROM llm_call_log
+          GROUP BY template_name
+          ORDER BY count DESC
+        `).all() as Array<{
+          template_name: string;
+          count: number;
+          total_cost: number;
+          min_cost: number;
+          max_cost: number;
+          avg_cost: number;
+          retry_count: number;
+          fail_count: number;
+        }>;
+
+        const types: LlmCallTypeStats[] = rows.map(row => ({
+          templateName: row.template_name,
+          label: TEMPLATE_LABELS[row.template_name] ?? row.template_name,
+          count: row.count,
+          totalCost: row.total_cost,
+          minCost: row.min_cost,
+          maxCost: row.max_cost,
+          avgCost: row.avg_cost,
+          retryCount: row.retry_count,
+          failCount: row.fail_count,
+          retryPct: row.count > 0 ? Math.round((row.retry_count / row.count) * 100) : 0,
+          failPct: row.count > 0 ? Math.round((row.fail_count / row.count) * 100) : 0,
+        }));
+
+        const totalCount = types.reduce((s, t) => s + t.count, 0);
+        const totalCost = types.reduce((s, t) => s + t.totalCost, 0);
+        const totalRetries = types.reduce((s, t) => s + t.retryCount, 0);
+        const totalFailures = types.reduce((s, t) => s + t.failCount, 0);
+
+        return { types, totalCount, totalCost, totalRetries, totalFailures, trackingEnabled: enabled };
+      } catch {
+        return { types: [], totalCount: 0, totalCost: 0, totalRetries: 0, totalFailures: 0, trackingEnabled: enabled };
+      }
+    },
+
+    isTrackingEnabled(): boolean {
+      return loadTrackingEnabled();
+    },
+
+    setTrackingEnabled(enabled: boolean): void {
+      trackingEnabled = enabled;
+      try {
+        const db = getDatabase();
+        db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES ('llm_tracking_enabled', ?)").run(enabled ? '1' : '0');
+      } catch {
+        // ignore
+      }
+    },
+  };
+}
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/llmStatsService.test.ts`
+Expected: PASS
+
+**Step 5: Commit**
+
+```bash
+git add server/src/llm/llmStatsService.ts server/src/llm/__tests__/llmStatsService.test.ts
+git commit -m "feat: add llmStatsService with SQLite persistence and aggregation"
+```
+
+---
+
+### Task 5: Instrument llmService to Record Calls
+
+**Files:**
+- Modify: `server/src/llm/llmService.ts`
+- Modify: `server/src/llm/__tests__/llmService.test.ts`
+
+**Step 1: Write test for stats recording integration**
+
+Add tests to `server/src/llm/__tests__/llmService.test.ts`:
+
+```typescript
+it('records successful call to stats service', async () => {
+  setupEnv();
+  globalThis.fetch = vi.fn().mockResolvedValue({
+    ok: true,
+    json: () => Promise.resolve({
+      choices: [{ message: { content: 'ok' } }],
+      usage: { prompt_tokens: 100, completion_tokens: 50 },
+    }),
+  });
+
+  const mockStatsService = {
+    record: vi.fn(),
+    getStats: vi.fn(),
+    isTrackingEnabled: vi.fn().mockReturnValue(true),
+    setTrackingEnabled: vi.fn(),
+  };
+
+  const service = createLlmService(undefined, mockStatsService);
+  await service.generate('backstory', { npcName: 'X', stats: 'S:1' });
+  await vi.advanceTimersByTimeAsync(100);
+
+  expect(mockStatsService.record).toHaveBeenCalledWith(
+    'backstory', 100, 50, 'free/model:free', 0, false, expect.any(Number),
+  );
+});
+
+it('records failed call to stats service', async () => {
+  setupEnv();
+  globalThis.fetch = vi.fn().mockResolvedValue({
+    ok: false,
+    status: 400,
+    statusText: 'Bad Request',
+  });
+
+  const mockStatsService = {
+    record: vi.fn(),
+    getStats: vi.fn(),
+    isTrackingEnabled: vi.fn().mockReturnValue(true),
+    setTrackingEnabled: vi.fn(),
+  };
+
+  const service = createLlmService(undefined, mockStatsService);
+  await service.generate('backstory', { npcName: 'X', stats: 'S:1' });
+  await vi.advanceTimersByTimeAsync(100);
+
+  expect(mockStatsService.record).toHaveBeenCalledWith(
+    'backstory', 0, 0, 'free/model:free', 0, true, expect.any(Number),
+  );
+});
+```
+
+**Step 2: Run test to verify it fails**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/llmService.test.ts`
+Expected: FAIL — `createLlmService` doesn't accept statsService parameter.
+
+**Step 3: Modify llmService to accept and use statsService**
+
+Modify `server/src/llm/llmService.ts`:
+
+Add import:
+```typescript
+import type { LlmStatsService } from './llmStatsService.js';
+```
+
+Change the `createLlmService` signature (line 21) to:
+```typescript
+export function createLlmService(logService?: LogService, statsService?: LlmStatsService): LlmService {
+```
+
+In the disabled-LLM early return (lines 24-35), no changes needed — stats won't be recorded.
+
+After the `isSuccess(result)` block in the `generate` method (around line 91-103), modify to capture stats. The key change is after `queue.enqueue()` returns, we need to record. Replace the section from `const result = await queue.enqueue(...)` through `return result;` with:
+
+```typescript
+      const result = await queue.enqueue({ ...rendered, model: currentModel });
+
+      if (isRateLimited(result)) {
+        logService?.log('warning', 'LLM', 'Rate limited, switching to fallback');
+        switchToFallback(result.resetAt);
+        statsService?.record(templateName, 0, 0, currentModel, 0, true, 0);
+        return null;
+      }
+
+      if (isSuccess(result)) {
+        counters.record(currentModel);
+        const inputTokens = result.usage?.promptTokens ?? 0;
+        const outputTokens = result.usage?.completionTokens ?? 0;
+        if (result.usage) {
+          tokenTracker.record(templateName, inputTokens, outputTokens);
+          console.log(tokenTracker.lastSummary());
+        }
+        statsService?.record(templateName, inputTokens, outputTokens, currentModel, result.retries, false, 0);
+        const totalRequests = Object.values(counters.getStats()).reduce((sum, s) => sum + s.total, 0);
+        if (totalRequests % 100 === 0) {
+          console.log(`[LLM] ${counters.getSummary()}`);
+          logService?.log('info', 'LLM', counters.getSummary());
+        }
+        return result.content;
+      }
+
+      // null result = failure
+      statsService?.record(templateName, 0, 0, currentModel, 0, true, 0);
+      return result;
+```
+
+**Step 4: Run test to verify it passes**
+
+Run: `npm -w server run test -- --run src/llm/__tests__/llmService.test.ts`
+Expected: PASS
+
+**Step 5: Run all tests**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 6: Commit**
+
+```bash
+git add server/src/llm/llmService.ts server/src/llm/__tests__/llmService.test.ts
+git commit -m "feat: instrument llmService to record calls via llmStatsService"
+```
+
+---
+
+### Task 6: Schema Migration — Add llm_call_log Table
+
+**Files:**
+- Modify: `server/src/persistence/database.ts`
+- Modify: `server/src/persistence/saveManager.ts`
+- Modify: `server/src/persistence/__tests__/database.test.ts`
+
+**Step 1: Write tests for migration**
+
+Add to `server/src/persistence/__tests__/database.test.ts`:
+
+```typescript
+it('CURRENT_SCHEMA_VERSION is 2', () => {
+  expect(CURRENT_SCHEMA_VERSION).toBe(2);
+});
+
+it('creates llm_call_log table in new databases', () => {
+  const dbPath = createTempDb();
+  openDatabase(dbPath);
+  const db = getDatabase();
+  const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[];
+  expect(tables.map(t => t.name)).toContain('llm_call_log');
+});
+
+it('creates llm_tracking_enabled metadata in new databases', () => {
+  const dbPath = createTempDb();
+  openDatabase(dbPath);
+  const db = getDatabase();
+  const row = db.prepare("SELECT value FROM metadata WHERE key = 'llm_tracking_enabled'").get() as { value: string };
+  expect(row.value).toBe('1');
+});
+```
+
+Note: Update the existing test `'CURRENT_SCHEMA_VERSION is 1'` to expect `2` instead. Also update `'sets schema_version in metadata'` test to expect `2`.
+
+**Step 2: Run tests to verify they fail**
+
+Run: `npm -w server run test -- --run src/persistence/__tests__/database.test.ts`
+Expected: FAIL — CURRENT_SCHEMA_VERSION still 1, no llm_call_log table.
+
+**Step 3: Update database.ts**
+
+In `server/src/persistence/database.ts`:
+
+Change line 3:
+```typescript
+export const CURRENT_SCHEMA_VERSION = 2;
+```
+
+Add to `SCHEMA_SQL` before the closing backtick (after the CREATE INDEX statements, around line 103):
+
+```sql
+
+CREATE TABLE IF NOT EXISTS llm_call_log (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  tick INTEGER,
+  template_name TEXT NOT NULL,
+  input_tokens INTEGER NOT NULL,
+  output_tokens INTEGER NOT NULL,
+  cost_usd REAL NOT NULL,
+  model TEXT NOT NULL,
+  retries INTEGER NOT NULL DEFAULT 0,
+  failed INTEGER NOT NULL DEFAULT 0,
+  timestamp TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_llm_call_log_template ON llm_call_log(template_name);
+```
+
+In the `openDatabase` function, after setting schema_version for new databases (line 118-119), also set tracking enabled:
+
+```typescript
+  if (!existing) {
+    db.prepare("INSERT INTO metadata (key, value) VALUES ('schema_version', ?)").run(String(CURRENT_SCHEMA_VERSION));
+    db.prepare("INSERT INTO metadata (key, value) VALUES ('llm_tracking_enabled', '1')").run();
+  }
+```
+
+**Step 4: Update saveManager.ts migration**
+
+In `server/src/persistence/saveManager.ts`, replace the migration placeholder in `openExistingSave()` (lines 30-36):
+
+```typescript
+    if (version < CURRENT_SCHEMA_VERSION) {
+      const db = getDatabase();
+      if (version < 2) {
+        db.exec(`
+          CREATE TABLE IF NOT EXISTS llm_call_log (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            tick INTEGER,
+            template_name TEXT NOT NULL,
+            input_tokens INTEGER NOT NULL,
+            output_tokens INTEGER NOT NULL,
+            cost_usd REAL NOT NULL,
+            model TEXT NOT NULL,
+            retries INTEGER NOT NULL DEFAULT 0,
+            failed INTEGER NOT NULL DEFAULT 0,
+            timestamp TEXT NOT NULL
+          );
+          CREATE INDEX IF NOT EXISTS idx_llm_call_log_template ON llm_call_log(template_name);
+        `);
+        db.prepare("INSERT OR IGNORE INTO metadata (key, value) VALUES ('llm_tracking_enabled', '1')").run();
+      }
+      db.prepare(
+        "UPDATE metadata SET value = ? WHERE key = 'schema_version'",
+      ).run(String(CURRENT_SCHEMA_VERSION));
+    }
+```
+
+**Step 5: Run tests to verify they pass**
+
+Run: `npm -w server run test -- --run src/persistence/__tests__/database.test.ts`
+Expected: PASS
+
+**Step 6: Run all tests**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 7: Commit**
+
+```bash
+git add server/src/persistence/database.ts server/src/persistence/saveManager.ts server/src/persistence/__tests__/database.test.ts
+git commit -m "feat: schema migration v1→v2 adding llm_call_log table"
+```
+
+---
+
+### Task 7: Wire llmStatsService into GameLoop
+
+**Files:**
+- Modify: `server/src/game/GameLoop.ts`
+
+**Step 1: Add llmStatsService to GameLoop**
+
+Add import to `server/src/game/GameLoop.ts`:
+```typescript
+import { createLlmStatsService, type LlmStatsService } from '../llm/llmStatsService.js';
+```
+
+Add property to the class (after `readonly llmService: LlmService;` on line 48):
+```typescript
+readonly llmStatsService: LlmStatsService;
+```
+
+In the constructor (after `this.llmService = createLlmService(this.logService);` on line 68), create and pass the stats service:
+```typescript
+    this.llmStatsService = createLlmStatsService();
+    this.llmService = createLlmService(this.logService, this.llmStatsService);
+```
+
+(Remove the old `this.llmService = createLlmService(this.logService);` line.)
+
+**Step 2: Run all tests**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 3: Commit**
+
+```bash
+git add server/src/game/GameLoop.ts
+git commit -m "feat: wire llmStatsService into GameLoop"
+```
+
+---
+
+### Task 8: Add Socket Events for LLM Stats
+
+**Files:**
+- Modify: `server/src/network/SocketServer.ts`
+
+**Step 1: Add socket handlers**
+
+In `server/src/network/SocketServer.ts`, after the `admin-reset-defaults` handler (around line 199), add:
+
+```typescript
+      socket.on('admin-llm-stats', () => {
+        if (!this.authenticatedSockets.has(socket.id)) return;
+        const stats = this.gameLoop.llmStatsService.getStats();
+        socket.emit('admin-llm-stats-result', stats);
+      });
+
+      socket.on('admin-toggle-llm-tracking', (data: { enabled: boolean }) => {
+        if (!this.authenticatedSockets.has(socket.id)) return;
+        this.gameLoop.llmStatsService.setTrackingEnabled(data.enabled);
+        this.io.emit('admin-llm-tracking-toggled', { enabled: data.enabled });
+        this.gameLoop.logService.log('info', 'Game', `Admin ${data.enabled ? 'enabled' : 'disabled'} LLM tracking`);
+      });
+```
+
+**Step 2: Run all tests**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 3: Commit**
+
+```bash
+git add server/src/network/SocketServer.ts
+git commit -m "feat: add socket handlers for LLM stats and tracking toggle"
+```
+
+---
+
+### Task 9: Add Client Socket Methods for LLM Stats
+
+**Files:**
+- Modify: `client/src/network/SocketClient.ts`
+
+**Step 1: Add callbacks and methods**
+
+Add to the callback declarations (after `onAdminConstantsUpdated` around line 31):
+
+```typescript
+  onLlmStatsResult: ((data: LlmStatsData) => void) | null = null;
+  onLlmTrackingToggled: ((data: { enabled: boolean }) => void) | null = null;
+```
+
+Add import for `LlmStatsData` in the import statement at line 2:
+```typescript
+import type { ..., LlmStatsData } from '@dflike/shared';
+```
+
+Add socket listeners (after line 94, after the `admin-constants-updated` listener):
+
+```typescript
+    this.socket.on('admin-llm-stats-result', (data) => this.onLlmStatsResult?.(data));
+    this.socket.on('admin-llm-tracking-toggled', (data) => this.onLlmTrackingToggled?.(data));
+```
+
+Add methods (after `adminResetDefaults()` around line 153):
+
+```typescript
+  requestLlmStats(): void {
+    this.socket.emit('admin-llm-stats');
+  }
+
+  toggleLlmTracking(enabled: boolean): void {
+    this.socket.emit('admin-toggle-llm-tracking', { enabled });
+  }
+```
+
+**Step 2: Commit**
+
+```bash
+git add client/src/network/SocketClient.ts
+git commit -m "feat: add client socket methods for LLM stats"
+```
+
+---
+
+### Task 10: Update AdminPanel UI — Two-Column Layout with LLM Analytics
+
+**Files:**
+- Modify: `client/src/ui/AdminPanel.ts`
+
+**Step 1: Rewrite AdminPanel for two-column layout**
+
+This is the largest UI change. The key modifications:
+
+1. After authentication, show two columns side by side
+2. Left column: existing constants editor (move into its own scrollable div)
+3. Right column: LLM tracking toggle + stats table
+4. Both columns independently scrollable
+
+Add import at top:
+```typescript
+import type { TunableConstants, TunableKey, LlmStatsData } from '@dflike/shared';
+```
+
+Add new callbacks and state to the class:
+```typescript
+  onRequestLlmStats: (() => void) | null = null;
+  onToggleLlmTracking: ((enabled: boolean) => void) | null = null;
+  private statsColumn: HTMLDivElement | null = null;
+  private trackingToggle: HTMLInputElement | null = null;
+```
+
+Modify the `editorView` construction. Change from a single column to a flex row with two scrollable columns. In `buildEditor(constants)`:
+
+After `this.editorView.innerHTML = '';`, change the editorView to flexbox row:
+```typescript
+    this.editorView.style.cssText = `
+      display: none;
+      flex-direction: row;
+      gap: 8px;
+      flex: 1;
+      overflow: hidden;
+    `;
+```
+
+Wrap existing constant inputs in a left column div:
+```typescript
+    const leftCol = document.createElement('div');
+    leftCol.style.cssText = `
+      flex: 1;
+      overflow-y: auto;
+      display: flex;
+      flex-direction: column;
+      gap: 4px;
+      min-width: 0;
+    `;
+```
+
+Put all existing constant rows and the reset button into `leftCol` instead of `this.editorView`.
+
+Create the right column:
+```typescript
+    const rightCol = document.createElement('div');
+    rightCol.style.cssText = `
+      flex: 1;
+      overflow-y: auto;
+      display: flex;
+      flex-direction: column;
+      gap: 4px;
+      min-width: 0;
+    `;
+    this.statsColumn = rightCol;
+```
+
+Build the tracking toggle at the top of rightCol:
+```typescript
+    const toggleRow = document.createElement('div');
+    toggleRow.style.cssText = `
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      padding: 4px 0;
+      border-bottom: 1px solid #1c1450;
+      flex-shrink: 0;
+    `;
+
+    const toggleLabel = document.createElement('span');
+    toggleLabel.style.cssText = 'color: #a0a0c0; font-size: 8px;';
+    toggleLabel.textContent = 'LLM Tracking';
+
+    this.trackingToggle = document.createElement('input');
+    this.trackingToggle.type = 'checkbox';
+    this.trackingToggle.checked = true;
+    this.trackingToggle.style.cssText = 'accent-color: #8878a8; cursor: pointer;';
+    this.trackingToggle.addEventListener('change', () => {
+      this.onToggleLlmTracking?.(this.trackingToggle!.checked);
+    });
+
+    toggleRow.appendChild(toggleLabel);
+    toggleRow.appendChild(this.trackingToggle);
+    rightCol.appendChild(toggleRow);
+```
+
+Add a placeholder for stats:
+```typescript
+    const statsPlaceholder = document.createElement('div');
+    statsPlaceholder.style.cssText = 'color: #555; font-size: 8px; padding: 8px 0; text-align: center;';
+    statsPlaceholder.textContent = 'Loading stats...';
+    rightCol.appendChild(statsPlaceholder);
+```
+
+Append both columns:
+```typescript
+    this.editorView.appendChild(leftCol);
+    this.editorView.appendChild(rightCol);
+```
+
+Add a method to request stats refresh (called when admin tab opens):
+```typescript
+  requestStatsRefresh(): void {
+    if (this.authenticated) {
+      this.onRequestLlmStats?.();
+    }
+  }
+```
+
+Add a method to update the stats display:
+```typescript
+  updateLlmStats(data: LlmStatsData): void {
+    if (!this.statsColumn) return;
+    if (this.trackingToggle) {
+      this.trackingToggle.checked = data.trackingEnabled;
+    }
+
+    // Remove everything after the toggle row
+    while (this.statsColumn.children.length > 1) {
+      this.statsColumn.removeChild(this.statsColumn.lastChild!);
+    }
+
+    if (data.types.length === 0) {
+      const empty = document.createElement('div');
+      empty.style.cssText = 'color: #555; font-size: 8px; padding: 8px 0; text-align: center;';
+      empty.textContent = 'No LLM calls recorded yet';
+      this.statsColumn.appendChild(empty);
+      return;
+    }
+
+    // Stats rows for each type
+    for (const type of data.types) {
+      const row = document.createElement('div');
+      row.style.cssText = `
+        display: flex;
+        flex-direction: column;
+        gap: 1px;
+        padding: 4px 0;
+        border-bottom: 1px solid #1c1450;
+      `;
+
+      const header = document.createElement('div');
+      header.style.cssText = `
+        display: flex;
+        justify-content: space-between;
+        align-items: baseline;
+      `;
+
+      const nameEl = document.createElement('span');
+      nameEl.style.cssText = 'color: #e0d0b0; font-size: 9px; font-weight: bold;';
+      nameEl.textContent = type.label;
+
+      const countEl = document.createElement('span');
+      countEl.style.cssText = 'color: #8878a8; font-size: 8px;';
+      countEl.textContent = `${type.count} calls`;
+
+      header.appendChild(nameEl);
+      header.appendChild(countEl);
+      row.appendChild(header);
+
+      const costLine = document.createElement('div');
+      costLine.style.cssText = 'color: #a0a0c0; font-size: 7px;';
+      const fmt = (n: number) => n < 0.01 ? `$${n.toFixed(6)}` : `$${n.toFixed(4)}`;
+      costLine.textContent = `min ${fmt(type.minCost)} / avg ${fmt(type.avgCost)} / max ${fmt(type.maxCost)}`;
+      row.appendChild(costLine);
+
+      const totalLine = document.createElement('div');
+      totalLine.style.cssText = `
+        display: flex;
+        justify-content: space-between;
+        font-size: 7px;
+      `;
+
+      const totalCostEl = document.createElement('span');
+      totalCostEl.style.cssText = 'color: #a0a0c0;';
+      totalCostEl.textContent = `total: ${fmt(type.totalCost)}`;
+
+      const ratesEl = document.createElement('span');
+      ratesEl.style.cssText = `color: ${type.failPct > 0 ? '#ff6666' : '#a0a0c0'};`;
+      ratesEl.textContent = `retry ${type.retryPct}% / fail ${type.failPct}%`;
+
+      totalLine.appendChild(totalCostEl);
+      totalLine.appendChild(ratesEl);
+      row.appendChild(totalLine);
+
+      this.statsColumn.appendChild(row);
+    }
+
+    // Totals row
+    const totalsRow = document.createElement('div');
+    totalsRow.style.cssText = `
+      display: flex;
+      flex-direction: column;
+      gap: 1px;
+      padding: 6px 0;
+      border-top: 2px solid #8878a8;
+      margin-top: 4px;
+    `;
+
+    const totalsHeader = document.createElement('div');
+    totalsHeader.style.cssText = `
+      display: flex;
+      justify-content: space-between;
+      align-items: baseline;
+    `;
+
+    const totalsLabel = document.createElement('span');
+    totalsLabel.style.cssText = 'color: #e0d0b0; font-size: 9px; font-weight: bold;';
+    totalsLabel.textContent = 'TOTAL';
+
+    const totalsCalls = document.createElement('span');
+    totalsCalls.style.cssText = 'color: #8878a8; font-size: 8px;';
+    totalsCalls.textContent = `${data.totalCount} calls`;
+
+    totalsHeader.appendChild(totalsLabel);
+    totalsHeader.appendChild(totalsCalls);
+    totalsRow.appendChild(totalsHeader);
+
+    const totalsCost = document.createElement('div');
+    totalsCost.style.cssText = 'color: #a0a0c0; font-size: 7px;';
+    const fmt = (n: number) => n < 0.01 ? `$${n.toFixed(6)}` : `$${n.toFixed(4)}`;
+    totalsCost.textContent = `total cost: ${fmt(data.totalCost)} / retries: ${data.totalRetries} / failures: ${data.totalFailures}`;
+    totalsRow.appendChild(totalsCost);
+
+    this.statsColumn.appendChild(totalsRow);
+  }
+```
+
+**Step 2: Commit**
+
+```bash
+git add client/src/ui/AdminPanel.ts
+git commit -m "feat: add two-column admin panel with LLM analytics display"
+```
+
+---
+
+### Task 11: Update LeftPanel Width for Admin Tab
+
+**Files:**
+- Modify: `client/src/ui/LeftPanel.ts`
+
+**Step 1: Dynamic width change**
+
+In the `expand()` method (line 324), add dynamic width logic. After setting `this.activeTab = tab;`:
+
+```typescript
+    const width = tab === 'admin' ? 680 : 340;
+    this.panel.style.width = `${width}px`;
+```
+
+In the `collapse()` method (line 341), update the translateX to use the current panel width:
+
+```typescript
+    this.panel.style.width = '340px';
+    this.container.style.transform = 'translateY(-50%) translateX(-340px)';
+```
+
+Also update the initial collapsed translateX in `showContent` or ensure `expand()` properly accounts for the new width. The `expand()` method on line 329 has:
+```typescript
+    this.container.style.transform = 'translateY(-50%) translateX(0)';
+```
+This is fine — translateX(0) works regardless of panel width.
+
+The `collapse()` method needs to match. Currently (line 343):
+```typescript
+    this.container.style.transform = 'translateY(-50%) translateX(-340px)';
+```
+
+This should dynamically use the panel width, but since we reset to 340px on collapse it's fine as-is.
+
+**Step 2: Commit**
+
+```bash
+git add client/src/ui/LeftPanel.ts
+git commit -m "feat: widen LeftPanel to 680px when admin tab is active"
+```
+
+---
+
+### Task 12: Wire Admin Panel LLM Stats in GameScene
+
+**Files:**
+- Modify: `client/src/scenes/GameScene.ts`
+
+**Step 1: Add socket callbacks and panel wiring**
+
+In the admin panel callbacks section (around line 305-327), add:
+
+```typescript
+    // LLM stats callbacks
+    this.adminPanel.onRequestLlmStats = () => {
+      this.client.requestLlmStats();
+    };
+    this.adminPanel.onToggleLlmTracking = (enabled) => {
+      this.client.toggleLlmTracking(enabled);
+    };
+
+    this.client.onLlmStatsResult = (data) => {
+      this.adminPanel.updateLlmStats(data);
+    };
+    this.client.onLlmTrackingToggled = (data) => {
+      // Refresh stats to reflect the change
+      this.client.requestLlmStats();
+    };
+```
+
+Also, trigger a stats refresh when the admin tab is opened. Listen for the left-panel-opened event or add it to the existing admin tab expand logic. Find where tabs are expanded and add:
+
+In the LeftPanel's `expand` method, it dispatches `left-panel-opened` with `{ tab }`. In GameScene, listen for this event to request stats when admin tab opens:
+
+```typescript
+    document.addEventListener('left-panel-opened', (e: Event) => {
+      const detail = (e as CustomEvent).detail;
+      if (detail?.tab === 'admin') {
+        this.adminPanel.requestStatsRefresh();
+      }
+    });
+```
+
+**Step 2: Commit**
+
+```bash
+git add client/src/scenes/GameScene.ts
+git commit -m "feat: wire LLM stats into GameScene admin panel"
+```
+
+---
+
+### Task 13: Run All Tests and Verify
+
+**Step 1: Run full test suite**
+
+Run: `npm -w server run test`
+Expected: All tests pass.
+
+**Step 2: Rebuild shared types**
+
+Run: `npx -w shared tsc`
+Expected: Clean build.
+
+**Step 3: Build client**
+
+Run: `npm -w client run build`
+Expected: Clean build, no errors.
+
+**Step 4: Commit any remaining changes**
+
+```bash
+git add -A
+git status
+# Only commit if there are changes
+```
+
+---
+
+### Task 14: Test Database Migration on Existing Save
+
+**Step 1: Start the server to trigger migration**
+
+Run: `npm -w server run dev`
+
+Watch for:
+- Server starts without errors
+- If existing save exists: migration runs (schema_version updates 1→2, llm_call_log table created)
+- If no save: fresh world created with schema v2
+
+**Step 2: Verify migration worked**
+
+While server is running (or after stopping), verify with:
+```bash
+sqlite3 server/saves/default.db "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
+```
+Expected: `llm_call_log` appears in the table list.
+
+```bash
+sqlite3 server/saves/default.db "SELECT value FROM metadata WHERE key = 'schema_version';"
+```
+Expected: `2`
+
+```bash
+sqlite3 server/saves/default.db "SELECT value FROM metadata WHERE key = 'llm_tracking_enabled';"
+```
+Expected: `1`
+
+**Step 3: Verify with fresh world**
+
+Run: `npm -w server run dev -- --new-world`
+Expected: New world created with schema v2 and llm_call_log table.
+
+**Step 4: Stop server and commit if needed**
+
+Ctrl-C to stop. Any final adjustments committed.