update the codebase poc ver1

This commit is contained in:
DatTT127
2026-07-07 15:54:17 +07:00
parent fed5f277f4
commit 1622dc8fc5
452 changed files with 83999 additions and 66328 deletions

View File

@@ -0,0 +1,156 @@
# Agent Tools — Component Smoke Tests
Test each Edge-LLM agent tool **before** wiring Gemma function-calling.
Reference implementation: `PILOT_PROJECT/tmp/test_endpoint_img.py` (Modal `/api/chat` + streaming + vision).
## Three layers
| Layer | What it tests | Where |
|-------|----------------|-------|
| **0** | Modal Ollama MedGemma (`/api/chat`, stream) | `smoke/modal-ollama.mjs` |
| **0b** | Exa direct (`exa_search`) | `smoke/exa-direct.mjs` |
| **0c** | Supabase direct (`supabase_query`) | `smoke/supabase-direct.mjs` |
| **1** | FastAPI BFF routes (`exa`, `supabase`, `embed`, `cloud-consult`) | `smoke/bff-tools.mjs` |
| **2** | `ToolExecutor` in browser (IndexedDB cache + offline queue) | gemma4_e2b **Tool smoke** panel |
Gemma agent loop = **Layer 3** (after 02 pass).
---
## Layer 0 — Modal Ollama (default URL baked in)
Uses the working deploy from `test_endpoint_img.py`:
```
https://dtj-tran--ollama-medgemma-ollamaserver-web.modal.run
```
```bash
cd ml/tests/agent_tools
npm run smoke:modal
```
Tests:
- `GET /api/tags``medgemma:4b` present
- `POST /api/chat` text (non-stream)
- `POST /api/chat` clinical prompt (**stream:true**, same as Python)
- `POST /api/chat` vision (**stream**, when image file exists)
- `escalate_medgemma` tool-shaped clinical prompt
### Vision test (optional)
```bash
export SMOKE_IMAGE_PATH="/path/to/ultrasound.png"
npm run smoke:modal
```
Default path (if file exists): `PILOT_PROJECT/tmp/other_angle/med_lat_2.png`
### Override endpoint
Accepts base URL **or** full path ending in `/api/chat`:
```bash
export MODAL_MEDGEMMA_ENDPOINT="https://dtj-tran--ollama-medgemma-ollamaserver-web.modal.run/api/chat"
npm run smoke:modal
```
---
## Layer 0b/0c — Exa + Supabase (direct, no BFF)
Auto-loads `PILOT_PROJECT/secrets/aws_secret/.env` when present.
```bash
cd ml/tests/agent_tools
npm run smoke:other # both
npm run smoke:exa # exa_search only
npm run smoke:supabase # supabase_query only
```
---
## Layer 1 — BFF agent routes
**Quick smoke BFF** (loads secrets, no GCP/Redis):
```bash
cd CODEBASE
conda activate vkist_ultra
PYTHONPATH=. python backend/tests/smoke_agent_tools_server.py
```
```bash
cd ml/tests/agent_tools
npm run smoke:bff
```
Or run full `backend.main` on port **8000** (not the CV test proxy on `:8001`):
```bash
cd CODEBASE
export MODAL_MEDGEMMA_ENDPOINT="https://dtj-tran--ollama-medgemma-ollamaserver-web.modal.run"
export EXA_API_KEY="..."
export SUPABASE_URL="https://<ref>.supabase.co"
export SUPABASE_SERVICE_ROLE_KEY="..."
export EMBED_QUERY_MOCK=1 # PoC until embedder wired
PYTHONPATH=. uvicorn backend.main:app --host 127.0.0.1 --port 8000
```
```bash
cd ml/tests/agent_tools
export BFF_BASE_URL=http://127.0.0.1:8000
npm run smoke:bff
```
> **Note:** `cloud_llm_gateway.py` currently calls `/api/generate`. Your Modal server supports both `/api/chat` and `/api/generate`. Layer 0 validates `/api/chat`; BFF `escalate_medgemma` uses `/api/generate` until backend is migrated.
### `escalate_medgemma` via BFF (optional)
```bash
export BFF_AUTH_TOKEN="<jwt>"
redis-cli SET "consent:tool-smoke-001" 1 EX 7200
redis-cli SET "session_owner:tool-smoke-001" "<user_id>" EX 7200
npm run smoke:bff
```
---
## Layer 2 — Browser ToolExecutor
```bash
cd ml/tests/gemma4_e2b && npm run dev # :5174
```
Sidebar → **Tool smoke (no Gemma)** → uncheck **Mock tools****Run all**.
---
## Run everything
```bash
cd ml/tests/agent_tools && npm run smoke
```
Layer 0 runs automatically with the default Modal URL.
---
## Tool → route map
| Tool | Route |
|------|-------|
| `exa_search` | `POST /api/v1/agent/tools/exa/search` |
| `supabase_query` | `POST /api/v1/agent/tools/supabase/query` |
| `escalate_medgemma` | `POST /api/v1/cloud-consult` → Modal Ollama |
## Shared modules
| File | Role |
|------|------|
| `smoke/modal-config.mjs` | Default URL, model, session, image path |
| `smoke/ollama-client.mjs` | `/api/chat` stream + base64 image encode |
| `smoke/_helpers.mjs` | Timeouts, clearer fetch errors, result printing |

View File

@@ -0,0 +1,137 @@
/** @typedef {{ name: string; ok: boolean; detail: string; ms?: number; skipped?: boolean }} SmokeResult */
const RESET = '\x1b[0m';
const GREEN = '\x1b[32m';
const RED = '\x1b[31m';
const DIM = '\x1b[2m';
const BOLD = '\x1b[1m';
export function env(name, fallback = '') {
const value = process.env[name];
return value === undefined || value === '' ? fallback : value;
}
export function requireEnv(name) {
const value = env(name);
if (!value) {
throw new Error(`Missing required env: ${name}`);
}
return value;
}
/**
* @param {SmokeResult} result
*/
export function printResult(result) {
const icon = result.skipped
? `${DIM}${RESET}`
: result.ok
? `${GREEN}${RESET}`
: `${RED}${RESET}`;
const timing = result.ms !== undefined ? ` ${DIM}(${result.ms}ms)${RESET}` : '';
console.log(`${icon} ${BOLD}${result.name}${RESET}${timing}`);
if (result.detail) {
console.log(` ${result.detail}`);
}
}
/**
* @param {SmokeResult[]} results
*/
export function printSummary(results) {
const skipped = results.filter((r) => r.skipped).length;
const passed = results.filter((r) => r.ok && !r.skipped).length;
const failed = results.filter((r) => !r.ok && !r.skipped).length;
const total = results.length;
console.log('');
console.log(
`${BOLD}Summary:${RESET} ${passed} passed, ${failed} failed` +
(skipped ? `, ${skipped} skipped` : '') +
` (${total} total)`,
);
if (failed > 0) {
process.exitCode = 1;
}
}
/**
* @param {string} name
* @param {() => Promise<string>} fn
* @returns {Promise<SmokeResult>}
*/
export async function runCheck(name, fn) {
const started = Date.now();
try {
const detail = await fn();
return { name, ok: true, detail, ms: Date.now() - started };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
return { name, ok: false, detail: message, ms: Date.now() - started };
}
}
export async function postJson(url, body, headers = {}, timeoutMs = 60_000) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json', ...headers },
body: JSON.stringify(body),
signal: controller.signal,
});
const text = await response.text();
let json;
try {
json = text ? JSON.parse(text) : null;
} catch {
json = { raw: text };
}
if (!response.ok) {
const detail = json?.detail ?? json?.raw ?? response.statusText;
throw new Error(
`HTTP ${response.status}: ${typeof detail === 'string' ? detail : JSON.stringify(detail)}`,
);
}
return json;
} catch (error) {
if (error instanceof Error && error.name === 'AbortError') {
throw new Error(`POST timed out after ${timeoutMs}ms: ${url}`);
}
if (error instanceof TypeError && String(error.message).includes('fetch failed')) {
throw new Error(`Cannot reach ${url} — is the server running?`);
}
throw error;
} finally {
clearTimeout(timer);
}
}
export async function getJson(url, headers = {}, timeoutMs = 30_000) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(url, { headers, signal: controller.signal });
const text = await response.text();
let json;
try {
json = text ? JSON.parse(text) : null;
} catch {
json = { raw: text };
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${text.slice(0, 240)}`);
}
return json;
} catch (error) {
if (error instanceof Error && error.name === 'AbortError') {
throw new Error(`GET timed out after ${timeoutMs}ms: ${url}`);
}
if (error instanceof TypeError && String(error.message).includes('fetch failed')) {
throw new Error(`Cannot reach ${url} — is the server running?`);
}
throw error;
} finally {
clearTimeout(timer);
}
}

View File

@@ -0,0 +1,125 @@
#!/usr/bin/env node
/**
* Layer 1 — FastAPI BFF agent tool routes (no Gemma).
*
* Start backend first (from CODEBASE root):
* MODAL_MEDGEMMA_ENDPOINT=... EXA_API_KEY=... SUPABASE_URL=... \\
* PYTHONPATH=. uvicorn backend.main:app --host 127.0.0.1 --port 8000
*
* Usage:
* BFF_BASE_URL=http://127.0.0.1:8000 npm run smoke:bff
*/
import { env, printResult, printSummary, runCheck, postJson } from './_helpers.mjs';
const bffBase = env('BFF_BASE_URL', 'http://127.0.0.1:8000').replace(/\/$/, '');
const sessionId = env('SMOKE_SESSION_ID', 'tool-smoke-001');
const authToken = env('BFF_AUTH_TOKEN', '');
const authHeaders = authToken ? { Authorization: `Bearer ${authToken}` } : {};
async function main() {
console.log(`${'\x1b[1m'}BFF agent tools smoke${'\x1b[0m'}${bffBase}`);
console.log(`session_id: ${sessionId}\n`);
const results = [];
results.push(
await runCheck('POST /api/v1/embed', async () => {
const data = await postJson(
`${bffBase}/api/v1/embed`,
{ text: 'synovitis grade 2 knee ultrasound', task: 'retrieval-query' },
authHeaders,
);
const dims = Array.isArray(data.vector) ? data.vector.length : 0;
if (dims !== 768) {
throw new Error(`Expected 768-d vector, got ${dims}`);
}
return `source=${data.source} model=${data.model}`;
}),
);
results.push(
await runCheck('POST /api/v1/agent/tools/exa/search', async () => {
const data = await postJson(
`${bffBase}/api/v1/agent/tools/exa/search`,
{
query: 'synovitis grading power doppler ultrasound',
type: 'auto',
numResults: 3,
session_id: sessionId,
},
authHeaders,
);
const hits = data.hits ?? [];
if (!Array.isArray(hits) || hits.length === 0) {
throw new Error('No Exa hits returned');
}
const first = hits[0];
return `hits=${hits.length} first=${first.title ?? first.url ?? first.id}`;
}),
);
results.push(
await runCheck('POST /api/v1/agent/tools/supabase/query', async () => {
const data = await postJson(
`${bffBase}/api/v1/agent/tools/supabase/query`,
{
rpc: 'match_semantic_chunks',
args: {
query_text: 'synovitis grade 2 knee ultrasound',
match_count: 3,
filter_book_ids: ['mor', 'oho'],
},
session_id: sessionId,
},
authHeaders,
);
const rows = data.rows ?? [];
if (!Array.isArray(rows)) {
throw new Error('rows missing from response');
}
if (rows.length === 0) {
return 'rows=0 (RPC ok — corpus may be empty or embedding mock)';
}
const first = rows[0];
return `rows=${rows.length} first_chunk=${first.chunk_id} sim=${first.similarity ?? 'n/a'}`;
}),
);
results.push(
authToken
? await runCheck('POST /api/v1/cloud-consult (escalate_medgemma path)', async () => {
const data = await postJson(
`${bffBase}/api/v1/cloud-consult`,
{
session_id: sessionId,
prompt: 'In one sentence: what does synovitis grade 2 imply on power Doppler?',
task_type: 'clinical_deep_reasoning',
stream: false,
},
authHeaders,
);
const text = (data.text ?? '').trim();
if (!text) {
throw new Error('Empty MedGemma text');
}
return `tier=${data.tier} text=${text.slice(0, 120)}`;
})
: {
name: 'POST /api/v1/cloud-consult (escalate_medgemma path)',
ok: true,
skipped: true,
detail: 'skipped — set BFF_AUTH_TOKEN + Redis consent/session_owner keys',
ms: 0,
},
);
for (const result of results) {
printResult(result);
}
printSummary(results);
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});

View File

@@ -0,0 +1,99 @@
#!/usr/bin/env node
/**
* Layer 0b — Direct Exa search (same API the BFF proxies).
*
* Usage:
* EXA_API_KEY=... npm run smoke:exa
* SECRETS_ENV_FILE=.../secrets/aws_secret/.env npm run smoke:exa
*/
import { loadSecretsEnv } from './load-secrets-env.mjs';
import { env, printResult, printSummary, runCheck } from './_helpers.mjs';
const EXA_SEARCH_URL = 'https://api.exa.ai/search';
async function main() {
const secrets = loadSecretsEnv();
if (secrets.loaded) {
console.log(`${'\x1b[2m'}Loaded env from ${secrets.path}${'\x1b[0m'}\n`);
}
const apiKey = env('EXA_API_KEY');
if (!apiKey) {
console.error('Missing EXA_API_KEY. Set it or SECRETS_ENV_FILE.');
process.exitCode = 1;
return;
}
console.log(`${'\x1b[1m'}Exa direct smoke${'\x1b[0m'}${EXA_SEARCH_URL}\n`);
const results = [];
results.push(
await runCheck('exa_search (auto, highlights)', async () => {
const response = await fetch(EXA_SEARCH_URL, {
method: 'POST',
headers: {
'x-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify({
query: 'synovitis grading power doppler ultrasound knee',
type: 'auto',
numResults: 5,
contents: { highlights: true },
}),
});
const text = await response.text();
let data;
try {
data = JSON.parse(text);
} catch {
throw new Error(`Invalid JSON (${response.status}): ${text.slice(0, 200)}`);
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${JSON.stringify(data).slice(0, 200)}`);
}
const hits = data.results ?? [];
if (hits.length === 0) {
throw new Error('No results returned');
}
const first = hits[0];
return `hits=${hits.length} first=${first.title ?? first.url ?? first.id}`;
}),
);
results.push(
await runCheck('exa_search (clinical domains filter)', async () => {
const response = await fetch(EXA_SEARCH_URL, {
method: 'POST',
headers: {
'x-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify({
query: 'rheumatoid arthritis synovitis ultrasound grading',
type: 'auto',
numResults: 3,
includeDomains: ['pubmed.ncbi.nlm.nih.gov', 'acrabstracts.org'],
contents: { highlights: true },
}),
});
const data = await response.json();
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${JSON.stringify(data).slice(0, 200)}`);
}
const hits = data.results ?? [];
return `hits=${hits.length}`;
}),
);
for (const result of results) {
printResult(result);
}
printSummary(results);
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});

View File

@@ -0,0 +1,46 @@
#!/usr/bin/env node
/**
* Optional: load KEY=VALUE lines from a local secrets file (never commit secrets).
* Set SECRETS_ENV_FILE to e.g. PILOT_PROJECT/secrets/aws_secret/.env
*/
import { readFileSync, existsSync } from 'node:fs';
import { resolve } from 'node:path';
import { env } from './_helpers.mjs';
const DEFAULT_SECRETS_CANDIDATES = [
resolve(process.cwd(), '../../../../../secrets/aws_secret/.env'),
resolve(process.cwd(), '../../../../../../secrets/aws_secret/.env'),
];
export function loadSecretsEnv() {
const explicit = env('SECRETS_ENV_FILE');
const candidates = explicit ? [resolve(explicit)] : DEFAULT_SECRETS_CANDIDATES;
const file = candidates.find((path) => existsSync(path));
if (!file) {
return { loaded: false, path: null };
}
const text = readFileSync(file, 'utf8');
for (const line of text.split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) {
continue;
}
const eq = trimmed.indexOf('=');
if (eq === -1) {
continue;
}
const key = trimmed.slice(0, eq).trim();
let value = trimmed.slice(eq + 1).trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
if (process.env[key] === undefined || process.env[key] === '') {
process.env[key] = value;
}
}
return { loaded: true, path: file };
}

View File

@@ -0,0 +1,35 @@
import { env } from './_helpers.mjs';
/** Working deploy from PILOT_PROJECT/tmp/test_endpoint_img.py */
export const DEFAULT_MODAL_BASE =
'https://dtj-tran--ollama-medgemma-ollamaserver-web.modal.run';
export const DEFAULT_MEDGEMMA_MODEL = 'medgemma:4b';
/** Default ultrasound fixture (same path as test_endpoint_img.py). */
export const DEFAULT_SMOKE_IMAGE_PATH =
'/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/tmp/other_angle/med_lat_2.png';
/**
* Accepts base URL or full path ending in /api/chat or /api/generate.
* @param {string} [raw]
*/
export function resolveModalBaseUrl(raw) {
const value = (raw ?? env('MODAL_MEDGEMMA_ENDPOINT', DEFAULT_MODAL_BASE)).trim();
if (!value) {
throw new Error('MODAL_MEDGEMMA_ENDPOINT is empty');
}
return value.replace(/\/$/, '').replace(/\/api\/(chat|generate)$/, '');
}
export function medgemmaModel() {
return env('MEDGEMMA_MODEL', DEFAULT_MEDGEMMA_MODEL);
}
export function smokeSessionId() {
return env('SMOKE_SESSION_ID', 'tool-smoke-001');
}
export function smokeImagePath() {
return env('SMOKE_IMAGE_PATH', DEFAULT_SMOKE_IMAGE_PATH);
}

View File

@@ -0,0 +1,161 @@
#!/usr/bin/env node
/**
* Layer 0 — Modal Ollama MedGemma (aligns with PILOT_PROJECT/tmp/test_endpoint_img.py).
*
* Uses /api/chat with stream:true (primary) plus /api/tags health check.
* Optional vision test when SMOKE_IMAGE_PATH points to a readable file.
*
* Usage:
* npm run smoke:modal
* SMOKE_IMAGE_PATH=/path/to/us.png npm run smoke:modal
*/
import { access, constants } from 'node:fs/promises';
import { printResult, printSummary, runCheck } from './_helpers.mjs';
import {
DEFAULT_MODAL_BASE,
medgemmaModel,
resolveModalBaseUrl,
smokeImagePath,
smokeSessionId,
} from './modal-config.mjs';
import {
chatOnce,
chatStream,
clinicalChatStream,
encodeImageFile,
listOllamaModels,
} from './ollama-client.mjs';
const CLINICAL_PROMPT =
'In one sentence, explain what power Doppler grade 2 synovitis means on knee ultrasound.';
const IMAGE_PROMPT = 'Describe this ultrasound image in detail. Focus on synovium and power Doppler signal.';
async function imagePathReadable(path) {
try {
await access(path, constants.R_OK);
return true;
} catch {
return false;
}
}
async function main() {
const base = resolveModalBaseUrl();
const model = medgemmaModel();
const sessionId = smokeSessionId();
console.log(`${'\x1b[1m'}Modal MedGemma smoke${'\x1b[0m'}`);
console.log(`base: ${base}`);
if (base === DEFAULT_MODAL_BASE) {
console.log(`${'\x1b[2m'}(default from test_endpoint_img.py — override with MODAL_MEDGEMMA_ENDPOINT)${'\x1b[0m'}`);
}
console.log(`model: ${model}`);
console.log(`session: ${sessionId}\n`);
const results = [];
results.push(
await runCheck('GET /api/tags', async () => {
const data = await listOllamaModels(base);
const names = (data.models ?? []).map((entry) => entry.name).join(', ') || '(none)';
const hasModel = (data.models ?? []).some((entry) => entry.name === model);
if (!hasModel) {
throw new Error(`Model ${model} not in tags: ${names}`);
}
return names;
}),
);
results.push(
await runCheck('POST /api/chat (text, non-stream)', async () => {
const data = await chatOnce(base, {
model,
messages: [{ role: 'user', content: 'Reply with exactly one word: READY' }],
});
const text = (data.message?.content ?? '').trim();
if (!text) {
throw new Error('Empty message.content');
}
return text.slice(0, 80);
}),
);
results.push(
await runCheck('POST /api/chat (clinical, stream)', async () => {
let preview = '';
const text = await clinicalChatStream(base, model, CLINICAL_PROMPT);
preview = text.slice(0, 160) + (text.length > 160 ? '…' : '');
if (text.length < 20) {
throw new Error(`Response too short: ${text}`);
}
return preview;
}),
);
const imagePath = smokeImagePath();
const hasImage = await imagePathReadable(imagePath);
if (hasImage) {
results.push(
await runCheck('POST /api/chat (vision, stream)', async () => {
const imgBase64 = await encodeImageFile(imagePath);
const text = await chatStream(
base,
{
model,
messages: [
{
role: 'user',
content: IMAGE_PROMPT,
images: [imgBase64],
},
],
options: { temperature: 0.1, num_predict: 384 },
},
{ timeoutMs: 180_000 },
);
if (text.length < 30) {
throw new Error(`Vision response too short: ${text}`);
}
return `${text.slice(0, 140)}`;
}),
);
} else {
results.push({
name: 'POST /api/chat (vision, stream)',
ok: true,
skipped: true,
detail: `skipped — set SMOKE_IMAGE_PATH (tried ${imagePath})`,
ms: 0,
});
}
results.push(
await runCheck('POST /api/chat (escalate_medgemma tool shape)', async () => {
const text = await clinicalChatStream(
base,
model,
`[session ${sessionId}] ${CLINICAL_PROMPT}`,
);
if (!text) {
throw new Error('Empty escalate-style response');
}
return `chars=${text.length} preview=${text.slice(0, 100)}`;
}),
);
for (const result of results) {
printResult(result);
}
printSummary(results);
if (results.every((entry) => entry.ok)) {
console.log(`\nBFF hint: export MODAL_MEDGEMMA_ENDPOINT="${base}"`);
}
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});

View File

@@ -0,0 +1,162 @@
import { readFile } from 'node:fs/promises';
import { access } from 'node:fs/promises';
import { constants } from 'node:fs';
const DEFAULT_TIMEOUT_MS = 120_000;
/**
* @param {string} imagePath
*/
export async function encodeImageFile(imagePath) {
await access(imagePath, constants.R_OK);
const bytes = await readFile(imagePath);
return bytes.toString('base64');
}
/**
* @param {string} url
* @param {RequestInit} init
* @param {number} timeoutMs
*/
async function fetchWithTimeout(url, init, timeoutMs = DEFAULT_TIMEOUT_MS) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(url, { ...init, signal: controller.signal });
return response;
} catch (error) {
if (error instanceof Error && error.name === 'AbortError') {
throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`);
}
throw error;
} finally {
clearTimeout(timer);
}
}
/**
* @param {string} baseUrl
*/
export async function listOllamaModels(baseUrl) {
const response = await fetchWithTimeout(`${baseUrl}/api/tags`, { method: 'GET' });
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${text.slice(0, 240)}`);
}
return JSON.parse(text);
}
/**
* Non-streaming /api/chat — mirrors Ollama chat API.
* @param {string} baseUrl
* @param {object} payload
*/
export async function chatOnce(baseUrl, payload) {
const response = await fetchWithTimeout(`${baseUrl}/api/chat`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ...payload, stream: false }),
});
const text = await response.text();
let json;
try {
json = text ? JSON.parse(text) : null;
} catch {
throw new Error(`Invalid JSON (${response.status}): ${text.slice(0, 240)}`);
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${JSON.stringify(json).slice(0, 240)}`);
}
return json;
}
/**
* Streaming /api/chat — line-delimited JSON chunks (same as test_endpoint_img.py).
* @param {string} baseUrl
* @param {object} payload
* @param {{ onToken?: (token: string) => void; timeoutMs?: number }} [options]
*/
export async function chatStream(baseUrl, payload, options = {}) {
const { onToken, timeoutMs = DEFAULT_TIMEOUT_MS } = options;
const response = await fetchWithTimeout(
`${baseUrl}/api/chat`,
{
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ ...payload, stream: true }),
},
timeoutMs,
);
if (!response.ok) {
const text = await response.text();
throw new Error(`HTTP ${response.status}: ${text.slice(0, 240)}`);
}
if (!response.body) {
throw new Error('No response body for streaming chat');
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
let buffer = '';
let full = '';
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() ?? '';
for (const line of lines) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
const chunk = JSON.parse(trimmed);
const token = chunk.message?.content ?? '';
if (token) {
full += token;
onToken?.(token);
}
}
}
if (buffer.trim()) {
const chunk = JSON.parse(buffer.trim());
const token = chunk.message?.content ?? '';
if (token) {
full += token;
}
}
return full.trim();
}
/**
* Build user message for escalate_medgemma-style clinical prompts.
* @param {string} prompt
* @param {string[]} [imagesBase64]
*/
export function buildUserMessage(prompt, imagesBase64) {
const message = { role: 'user', content: prompt };
if (imagesBase64 && imagesBase64.length > 0) {
message.images = imagesBase64;
}
return message;
}
/**
* @param {string} baseUrl
* @param {string} model
* @param {string} prompt
* @param {string[]} [imagesBase64]
*/
export async function clinicalChatStream(baseUrl, model, prompt, imagesBase64) {
return chatStream(baseUrl, {
model,
messages: [buildUserMessage(prompt, imagesBase64)],
options: { temperature: 0.1, num_predict: 256 },
});
}

View File

@@ -0,0 +1,38 @@
#!/usr/bin/env node
/**
* Smoke exa_search + supabase_query (direct APIs, no Gemma).
*/
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
function runNode(script) {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, [path.join(__dirname, script)], {
stdio: 'inherit',
env: process.env,
});
child.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`${script} exited with code ${code}`));
}
});
});
}
async function main() {
console.log('=== Other agent tools (exa + supabase) ===\n');
console.log('--- exa_search ---\n');
await runNode('exa-direct.mjs');
console.log('\n--- supabase_query ---\n');
await runNode('supabase-direct.mjs');
}
main().catch((error) => {
console.error(error.message);
process.exitCode = 1;
});

View File

@@ -0,0 +1,48 @@
#!/usr/bin/env node
/**
* Run all agent-tool smoke layers in order:
* 0. Modal Ollama (default URL from test_endpoint_img.py)
* 1. BFF routes (BFF_BASE_URL)
*/
import { spawn } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import path from 'node:path';
import { DEFAULT_MODAL_BASE } from './modal-config.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
function runNode(script) {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, [path.join(__dirname, script)], {
stdio: 'inherit',
env: process.env,
});
child.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`${script} exited with code ${code}`));
}
});
});
}
async function main() {
console.log('=== Agent tools component smoke ===\n');
if (!process.env.MODAL_MEDGEMMA_ENDPOINT) {
process.env.MODAL_MEDGEMMA_ENDPOINT = DEFAULT_MODAL_BASE;
console.log(`MODAL_MEDGEMMA_ENDPOINT not set — using default:\n ${DEFAULT_MODAL_BASE}\n`);
}
console.log('--- Layer 0: Modal Ollama ---\n');
await runNode('modal-ollama.mjs');
console.log('\n--- Layer 1: BFF agent routes ---\n');
await runNode('bff-tools.mjs');
}
main().catch((error) => {
console.error(error.message);
process.exitCode = 1;
});

View File

@@ -0,0 +1,156 @@
#!/usr/bin/env node
/**
* Layer 0c — Direct Supabase knowledge RPC (same path BFF uses).
*
* Usage:
* SUPABASE_URL=... SUPABASE_SERVICE_ROLE_KEY=... npm run smoke:supabase
*/
import { createHash } from 'node:crypto';
import { loadSecretsEnv } from './load-secrets-env.mjs';
import { env, printResult, printSummary, runCheck } from './_helpers.mjs';
function deterministicEmbed(text, dimensions = 768) {
const vec = new Array(dimensions).fill(0);
const normalized = text.toLowerCase().trim();
if (!normalized) {
return vec;
}
for (let index = 0; index < normalized.length; index += 1) {
const code = normalized.charCodeAt(index);
const bucket = (code * (index + 17)) % dimensions;
vec[bucket] += 1;
}
const digest = createHash('sha256').update(normalized).digest();
for (let i = 0; i < dimensions; i += 1) {
vec[i] += digest[i % digest.length] / 255;
}
const norm = Math.sqrt(vec.reduce((sum, value) => sum + value * value, 0));
if (norm === 0) {
return vec;
}
return vec.map((value) => value / norm);
}
async function supabaseRpc(baseUrl, serviceKey, rpc, body) {
const response = await fetch(`${baseUrl.replace(/\/$/, '')}/rest/v1/rpc/${rpc}`, {
method: 'POST',
headers: {
apikey: serviceKey,
Authorization: `Bearer ${serviceKey}`,
'Content-Type': 'application/json',
Accept: 'application/json',
'Accept-Profile': 'knowledge',
'Content-Profile': 'knowledge',
},
body: JSON.stringify(body),
});
const text = await response.text();
let json;
try {
json = text ? JSON.parse(text) : null;
} catch {
json = { raw: text };
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${typeof json === 'string' ? json : JSON.stringify(json).slice(0, 240)}`);
}
return json;
}
async function main() {
const secrets = loadSecretsEnv();
if (secrets.loaded) {
console.log(`${'\x1b[2m'}Loaded env from ${secrets.path}${'\x1b[0m'}\n`);
}
const baseUrl = env('SUPABASE_URL');
const serviceKey = env('SUPABASE_SERVICE_ROLE_KEY');
if (!baseUrl || !serviceKey) {
console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY.');
process.exitCode = 1;
return;
}
console.log(`${'\x1b[1m'}Supabase direct smoke${'\x1b[0m'}${baseUrl}\n`);
const queryText = 'synovitis grade 2 knee ultrasound power doppler';
const results = [];
results.push(
await runCheck('match_semantic_chunks (deterministic embed)', async () => {
const embedding = deterministicEmbed(`task: search result | query: ${queryText}`);
const rows = await supabaseRpc(baseUrl, serviceKey, 'match_semantic_chunks', {
query_embedding: embedding,
match_count: 5,
filter_book_ids: ['mor', 'oho'],
});
if (!Array.isArray(rows)) {
throw new Error('Expected array response');
}
if (rows.length === 0) {
return 'rows=0 (RPC ok — try real EmbeddingGemma embed for quality)';
}
const first = rows[0];
return `rows=${rows.length} first=${first.chunk_id} book=${first.book_id} sim=${Number(first.similarity).toFixed(3)}`;
}),
);
{
const citationStarted = Date.now();
const chunkRows = await supabaseRpc(baseUrl, serviceKey, 'match_semantic_chunks', {
query_embedding: deterministicEmbed(`task: search result | query: ${queryText}`),
match_count: 1,
});
if (!Array.isArray(chunkRows) || chunkRows.length === 0) {
results.push({
name: 'get_corpus_citation (optional RPC)',
ok: true,
skipped: true,
detail: 'skipped — no chunk to cite',
ms: Date.now() - citationStarted,
});
} else {
try {
const citation = await supabaseRpc(baseUrl, serviceKey, 'get_corpus_citation', {
chunk_id: chunkRows[0].chunk_id,
});
const title = citation?.parent_title ?? citation?.book_id ?? chunkRows[0].chunk_id;
results.push({
name: 'get_corpus_citation (optional RPC)',
ok: true,
detail: `citation ok · ${title}`,
ms: Date.now() - citationStarted,
});
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (message.includes('PGRST202') || message.includes('404')) {
results.push({
name: 'get_corpus_citation (optional RPC)',
ok: true,
skipped: true,
detail:
'skipped — get_corpus_citation RPC not deployed yet (match_semantic_chunks is enough)',
ms: Date.now() - citationStarted,
});
} else {
results.push({
name: 'get_corpus_citation (optional RPC)',
ok: false,
detail: message,
ms: Date.now() - citationStarted,
});
}
}
}
}
for (const result of results) {
printResult(result);
}
printSummary(results);
}
main().catch((error) => {
console.error(error);
process.exitCode = 1;
});

View File

@@ -0,0 +1,7 @@
node_modules/
dist/
models/
*.litertlm
*.task
results/
public/mediapipe-wasm/

View File

@@ -0,0 +1,81 @@
# Gemma 4 E2B Decode Experiment
Browser-side experiment for **Gemma 4 E2B** using:
- **MediaPipe** `@mediapipe/tasks-genai` (WASM runtime + WebGPU backend)
- **OPFS** (Origin Private File System) for on-device model caching
- **Dedicated Web Worker** for isolated inference
This lives under `ml/tests/` as R&D before wiring into the frontend `llm.worker.ts`.
## Prerequisites
- Chrome or Edge desktop with **WebGPU** enabled
- Node.js 20+
- Network access for the **first** model download (~multi-GB)
- Use `http://localhost:5174` (not `127.0.0.1`) so OPFS cache is stable across sessions
## Quick start
```bash
cd PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/ml/tests/gemma4_e2b
npm install # also runs sync-wasm (copies MediaPipe WASM to public/)
npm run dev
```
Open http://localhost:5174
> **No model format matched?** Usually means OPFS has the wrong file (e.g. `.litertlm` instead of `.task`), a truncated download, or an HTML error page saved as the model. Click **Reset OPFS cache**, re-download `gemma-4-E2B-it-web.task`, hard-refresh, then load again. Requires `@mediapipe/tasks-genai` ≥ 0.10.27 WASM (`npm run sync-wasm`).
> **ModuleFactory not set?** The LLM runs in an ES-module Web Worker, which cannot use `importScripts`. This project syncs WASM from `node_modules` to `public/mediapipe-wasm/` and bootstraps `ModuleFactory` manually. If you see that error, run `npm run sync-wasm` and hard-refresh the page.
## Experiment flow
1. **Environment** — confirms WebGPU, OPFS, secure context
2. **Download model to OPFS** — fetches `gemma-4-E2B-it-web.task` (MediaPipe bundle) from HuggingFace once
3. **Initialize worker** — loads MediaPipe WASM + model stream from OPFS
4. **Run prompt / benchmark** — streams tokens, records TTFT and throughput
5. **Export results JSON** — saves decode metrics for comparison runs
## Model source
Default URL:
`https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/main/gemma-4-E2B-it-web.task`
> Use the **`.task`** file — this is the MediaPipe bundle format required by `@mediapipe/tasks-genai`.
Manifest written to OPFS: `models/gemma-4-E2B-it-web.manifest.json`
## Decode parameters
| Param | Default | Notes |
|-------|---------|-------|
| `maxTokens` | 512 | Input + output token budget |
| `topK` | 40 | Sampling breadth |
| `temperature` | 0.8 | Randomness |
| `randomSeed` | 101 | Reproducibility |
## Project layout
```
gemma4_e2b/
├── src/
│ ├── main.ts # UI + benchmark orchestration
│ ├── workers/llm.worker.ts # MediaPipe inference (isolated)
│ └── lib/
│ ├── capabilityProbe.ts
│ ├── opfsModelStore.ts
│ ├── decodeBenchmark.ts
│ └── prompts.ts
└── index.html
```
## Next steps
- Port proven worker contract to `frontend/implementation/src/workers/llm.worker.ts`
- Layer conformal decoding from `ml/spec/conformal_decoding_llm_with_domain_stritcly_constrain.md`
## References
- [MediaPipe LLM Inference Web](https://developers.google.com/edge/mediapipe/solutions/genai/llm_inference/web_js)

View File

@@ -0,0 +1,14 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Gemma 4 E2B Chat</title>
<link rel="icon" href="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 100 100'><text y='.9em' font-size='90'>G</text></svg>" />
<link rel="stylesheet" href="/src/styles.css" />
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/main.ts"></script>
</body>
</html>

View File

@@ -0,0 +1,17 @@
import { cpSync, existsSync, mkdirSync, rmSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const src = join(root, 'node_modules/@mediapipe/tasks-genai/wasm');
const dest = join(root, 'public/mediapipe-wasm');
if (!existsSync(src)) {
console.warn('[sync-mediapipe-wasm] Skip: run npm install first.');
process.exit(0);
}
rmSync(dest, { recursive: true, force: true });
mkdirSync(dest, { recursive: true });
cpSync(src, dest, { recursive: true });
console.log('[sync-mediapipe-wasm] Copied WASM assets to public/mediapipe-wasm');

View File

@@ -0,0 +1,26 @@
/// <reference types="vite/client" />
export {};
declare global {
interface Navigator {
gpu?: GPU;
}
interface GPU {
requestAdapter(): Promise<GPUAdapter | null>;
}
interface GPUAdapter {
requestAdapterInfo?(): Promise<GPUAdapterInfo>;
}
interface GPUAdapterInfo {
device?: string;
description?: string;
}
// MediaPipe WASM bootstrap (required in module Web Workers).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
var ModuleFactory: ((moduleArg?: Record<string, unknown>) => Promise<any>) | undefined;
}

View File

@@ -0,0 +1,102 @@
import {
AgentOrchestrator,
DEFAULT_AGENT_LOOP_CONFIG,
type AgentEvent,
type AgentExpectedOutputKind,
type AgentRunTurnResult,
} from '@vkist/agent-runtime';
import type { DecodeParams, GenerationStats, PromptOptions } from './types';
export interface AgentChatDeps {
llmClient: {
generateRawConversation(
conversationPrompt: string,
promptOptions: PromptOptions,
decode: DecodeParams,
onToken?: (partial: string) => void,
): Promise<{ rawOutput: string; stats: GenerationStats }>;
};
baseSystemPrompt: string;
readPromptOptions: () => PromptOptions;
readDecodeParams: () => DecodeParams;
mockTools?: boolean;
bffBaseUrl?: string;
}
function mergeDecode(base: DecodeParams, override?: Partial<DecodeParams>): DecodeParams {
if (!override) {
return base;
}
return {
// Never shrink the context window — MediaPipe maxTokens covers input + output.
maxTokens:
override.maxTokens !== undefined
? Math.max(base.maxTokens, override.maxTokens)
: base.maxTokens,
topK: override.topK ?? base.topK,
temperature: override.temperature ?? base.temperature,
randomSeed: override.randomSeed ?? base.randomSeed,
};
}
export async function runAgentChatTurn(
deps: AgentChatDeps,
input: { sessionId: string; userMessage: string },
onEvent?: (event: AgentEvent) => void,
signal?: AbortSignal,
): Promise<AgentRunTurnResult> {
const orchestrator = new AgentOrchestrator({
baseSystemPrompt: deps.baseSystemPrompt,
config: {
...DEFAULT_AGENT_LOOP_CONFIG,
mockTools: deps.mockTools ?? true,
bffBaseUrl: deps.bffBaseUrl ?? DEFAULT_AGENT_LOOP_CONFIG.bffBaseUrl,
},
onEvent: (event) => {
onEvent?.(event);
},
generateStep: async ({ conversationPrompt, chainOfThought, expectedKind, decodeOverride }) => {
if (signal?.aborted) {
throw new DOMException('Agent turn aborted', 'AbortError');
}
const promptOptions: PromptOptions = {
...deps.readPromptOptions(),
chainOfThought,
};
const decode = mergeDecode(deps.readDecodeParams(), decodeOverride);
const onToken =
expectedKind === 'final'
? (partial: string) => {
onEvent?.({ type: 'final_token', partial });
}
: undefined;
const { rawOutput } = await deps.llmClient.generateRawConversation(
conversationPrompt,
promptOptions,
decode,
onToken,
);
if (signal?.aborted) {
throw new DOMException('Agent turn aborted', 'AbortError');
}
return { rawOutput };
},
});
const promptOptions = deps.readPromptOptions();
return orchestrator.runTurn(
{
sessionId: input.sessionId,
userMessage: input.userMessage,
chainOfThought: promptOptions.chainOfThought,
},
signal,
);
}
export type { AgentEvent, AgentExpectedOutputKind, AgentRunTurnResult };

View File

@@ -0,0 +1,48 @@
import type { CapabilityReport } from './types';
export async function probeCapabilities(): Promise<CapabilityReport> {
const notes: string[] = [];
const webgpu = typeof navigator !== 'undefined' && 'gpu' in navigator && !!navigator.gpu;
const opfs =
typeof navigator !== 'undefined' &&
!!navigator.storage &&
typeof navigator.storage.getDirectory === 'function';
const worker = typeof Worker !== 'undefined';
const secureContext = typeof window !== 'undefined' && window.isSecureContext;
if (!webgpu) {
notes.push('WebGPU is unavailable. Gemma 4 E2B browser inference requires WebGPU (Chrome/Edge desktop).');
}
if (!opfs) {
notes.push('OPFS is unavailable. Model caching on-device will not work in this browser.');
}
if (!secureContext) {
notes.push('Not a secure context. Use https:// or http://localhost.');
}
if (location.hostname === '127.0.0.1') {
notes.push('OPFS is origin-scoped: 127.0.0.1 and localhost use separate storage.');
}
const ready = webgpu && opfs && worker && secureContext;
return { webgpu, opfs, worker, secureContext, ready, notes };
}
export async function probeWebGpuAdapterLabel(): Promise<string | null> {
if (!navigator.gpu) {
return null;
}
try {
const adapter = await navigator.gpu.requestAdapter();
if (!adapter) {
return 'no adapter';
}
if ('requestAdapterInfo' in adapter && typeof adapter.requestAdapterInfo === 'function') {
const info = await adapter.requestAdapterInfo();
return info.device || info.description || 'webgpu adapter';
}
return 'webgpu adapter';
} catch {
return 'probe failed';
}
}

View File

@@ -0,0 +1,49 @@
import type { DecodeParams, DecodeRun, ModelRuntime } from './types';
export function createRunId(): string {
return `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
}
export function estimateCharsPerSec(outputChars: number, totalMs: number): number {
if (totalMs <= 0) {
return 0;
}
return Number(((outputChars / totalMs) * 1000).toFixed(2));
}
export function buildDecodeRun(args: {
runId: string;
promptId: string;
modelSource: 'network' | 'opfs';
modelRuntime: ModelRuntime;
initMs: number;
ttftMs: number | null;
totalMs: number;
outputText: string;
decode: DecodeParams;
}): DecodeRun {
const outputChars = args.outputText.length;
return {
run_id: args.runId,
prompt_id: args.promptId,
model_source: args.modelSource,
model_runtime: args.modelRuntime,
init_ms: args.initMs,
ttft_ms: args.ttftMs,
total_ms: args.totalMs,
output_chars: outputChars,
chars_per_sec: estimateCharsPerSec(outputChars, args.totalMs),
decode: args.decode,
output_preview: args.outputText.slice(0, 280),
};
}
export function downloadResultsJson(runs: DecodeRun[]): void {
const blob = new Blob([JSON.stringify(runs, null, 2)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const anchor = document.createElement('a');
anchor.href = url;
anchor.download = `decode_runs_${Date.now()}.json`;
anchor.click();
URL.revokeObjectURL(url);
}

View File

@@ -0,0 +1,63 @@
import { FilesetResolver } from '@mediapipe/tasks-genai';
import { MEDIAPIPE_WASM_ROOT } from './types';
const MEDIAPIPE_WASM_VERSION = '0.10.28';
function withCacheBust(path: string): string {
const joiner = path.includes('?') ? '&' : '?';
return `${path}${joiner}v=${MEDIAPIPE_WASM_VERSION}`;
}
/**
* Load MediaPipe WASM bootstrap in a module Web Worker.
* Vite forbids `import()` from /public — fetch + blob URL avoids that.
* `importScripts` is also unavailable in module workers.
*/
async function bootstrapModuleFactory(loaderPath: string): Promise<void> {
const response = await fetch(withCacheBust(loaderPath));
if (!response.ok) {
throw new Error(`Failed to fetch WASM loader (${response.status}): ${loaderPath}`);
}
const source = await response.text();
const isEsModule =
loaderPath.includes('_module_') || /\bexport\s+default\b/.test(source);
if (isEsModule) {
const blob = new Blob([source], { type: 'text/javascript' });
const blobUrl = URL.createObjectURL(blob);
try {
const wasmModule = await import(/* @vite-ignore */ blobUrl);
globalThis.ModuleFactory = wasmModule.default ?? wasmModule.ModuleFactory;
} finally {
URL.revokeObjectURL(blobUrl);
}
return;
}
// Classic Emscripten UMD loader — eval in global worker scope sets ModuleFactory.
(0, eval)(source);
}
/** MediaPipe ES-module workers cannot use importScripts; bootstrap ModuleFactory manually. */
export async function resolveGenAiWasmFileset(): Promise<Awaited<ReturnType<typeof FilesetResolver.forGenAiTasks>>> {
const fileset = await FilesetResolver.forGenAiTasks(MEDIAPIPE_WASM_ROOT, true);
if (!globalThis.ModuleFactory) {
await bootstrapModuleFactory(fileset.wasmLoaderPath);
}
if (!globalThis.ModuleFactory) {
throw new Error(
'ModuleFactory not set. Run npm run sync-wasm, then hard-refresh the page.',
);
}
// Loader already executed; skip MediaPipe's importScripts path during createFromOptions.
return {
wasmBinaryPath: fileset.wasmBinaryPath,
assetLoaderPath: fileset.assetLoaderPath,
assetBinaryPath: fileset.assetBinaryPath,
wasmLoaderPath: '',
};
}

View File

@@ -0,0 +1,28 @@
import type { ChatSession } from './types';
import { CHAT_SESSIONS_STORE, withStore } from './memoryDb';
export async function saveChatSession(session: ChatSession): Promise<void> {
await withStore(CHAT_SESSIONS_STORE, 'readwrite', (store) => store.put(session));
}
export async function loadChatSession(sessionId: string): Promise<ChatSession | null> {
try {
const result = await withStore<ChatSession | undefined>(CHAT_SESSIONS_STORE, 'readonly', (store) =>
store.get(sessionId),
);
return result ?? null;
} catch {
return null;
}
}
export function createEmptySession(sessionId: string): ChatSession {
const now = Date.now();
return {
sessionId,
mode: 'ask',
messages: [],
createdAt: now,
updatedAt: now,
};
}

View File

@@ -0,0 +1,44 @@
import type { RetrievedEpisode } from './episodeRetriever';
import type { StoredChatMessage } from './types';
const MAX_HISTORY_TURNS = 6;
export function formatEpisodeContextBlock(retrieved: RetrievedEpisode[]): string {
if (retrieved.length === 0) {
return '';
}
const lines = retrieved.map(({ episode, score }, index) => {
const date = new Date(episode.createdAt).toISOString().slice(0, 10);
const facts =
episode.keyFacts.length > 0
? `\n Facts: ${episode.keyFacts.slice(0, 3).join('; ')}`
: '';
return `${index + 1}. [${date}] ${episode.title} (relevance ${score.toFixed(2)})\n ${episode.summary.slice(0, 280)}${facts}`;
});
return (
'Relevant prior episodes (episodic memory):\n' +
lines.join('\n') +
'\nUse only when helpful; do not invent facts not supported above.\n'
);
}
export function buildHistoryTurns(messages: StoredChatMessage[]): Array<{ role: 'user' | 'assistant'; text: string }> {
const turns = messages
.filter((msg) => msg.role === 'user' || msg.role === 'assistant')
.map((msg) => ({
role: msg.role as 'user' | 'assistant',
text: msg.text.trim(),
}))
.filter((turn) => turn.text.length > 0);
return turns.slice(-MAX_HISTORY_TURNS * 2);
}
export function mergeSystemPromptWithMemory(baseSystemPrompt: string, episodeBlock: string): string {
if (!episodeBlock.trim()) {
return baseSystemPrompt;
}
return `${baseSystemPrompt.trim()}\n\n${episodeBlock.trim()}`;
}

View File

@@ -0,0 +1,51 @@
import { EMBEDDING_DIMENSIONS } from './types';
/** PoC fallback when EmbeddingGemma BFF is unavailable — deterministic, L2-normalized. */
export async function deterministicEmbed(text: string, dimensions = EMBEDDING_DIMENSIONS): Promise<number[]> {
const vec = new Array<number>(dimensions).fill(0);
const normalized = text.toLowerCase().trim();
if (!normalized) {
return vec;
}
for (let i = 0; i < normalized.length; i += 1) {
const code = normalized.charCodeAt(i);
const idx = (code * (i + 17)) % dimensions;
vec[idx] += 1;
}
const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(normalized));
const bytes = new Uint8Array(digest);
for (let i = 0; i < dimensions; i += 1) {
vec[i] += bytes[i % bytes.length] / 255;
}
let norm = 0;
for (const value of vec) {
norm += value * value;
}
norm = Math.sqrt(norm);
if (norm === 0) {
return vec;
}
return vec.map((value) => value / norm);
}
export function cosineSimilarity(a: number[], b: number[]): number {
if (a.length !== b.length || a.length === 0) {
return 0;
}
let dot = 0;
let normA = 0;
let normB = 0;
for (let i = 0; i < a.length; i += 1) {
dot += a[i] * b[i];
normA += a[i] * a[i];
normB += b[i] * b[i];
}
const denom = Math.sqrt(normA) * Math.sqrt(normB);
if (denom === 0) {
return 0;
}
return dot / denom;
}

View File

@@ -0,0 +1,74 @@
import { EMBEDDING_DIMENSIONS, EMBEDDING_MODEL_ID } from './types';
import { deterministicEmbed } from './deterministicEmbed';
export type EmbedTask = 'retrieval-query' | 'retrieval-document';
export interface EmbedResult {
vector: number[];
model: string;
source: 'bff' | 'deterministic';
}
function formatEmbedInput(text: string, task: EmbedTask, title?: string): string {
if (task === 'retrieval-document') {
const titleValue = title?.trim() ? title.trim() : 'none';
return `title: ${titleValue} | text: ${text}`;
}
return `task: search result | query: ${text}`;
}
export async function embedText(
text: string,
task: EmbedTask,
options: { bffBaseUrl?: string; title?: string } = {},
): Promise<EmbedResult> {
const bffBaseUrl = options.bffBaseUrl ?? import.meta.env.VITE_API_BASE_URL ?? 'http://localhost:8000';
try {
const response = await fetch(`${bffBaseUrl.replace(/\/$/, '')}/api/v1/embed`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
text,
task,
title: options.title ?? null,
}),
});
if (response.ok) {
const payload = (await response.json()) as { vector: number[]; model?: string };
if (Array.isArray(payload.vector) && payload.vector.length === EMBEDDING_DIMENSIONS) {
return {
vector: payload.vector,
model: payload.model ?? EMBEDDING_MODEL_ID,
source: 'bff',
};
}
}
} catch {
// Fall through to deterministic embed for offline PoC.
}
return {
vector: await deterministicEmbed(formatEmbedInput(text, task, options.title), EMBEDDING_DIMENSIONS),
model: `${EMBEDDING_MODEL_ID}-deterministic`,
source: 'deterministic',
};
}
export async function embedQuery(text: string, bffBaseUrl?: string): Promise<EmbedResult> {
return embedText(text, 'retrieval-query', { bffBaseUrl });
}
export async function embedDocument(
text: string,
title?: string,
bffBaseUrl?: string,
): Promise<EmbedResult> {
return embedText(text, 'retrieval-document', { title, bffBaseUrl });
}
export function buildEpisodeEmbedText(title: string, summary: string, keyFacts: string[]): string {
const facts = keyFacts.length > 0 ? `\n${keyFacts.map((f) => `- ${f}`).join('\n')}` : '';
return `${summary}${facts}`.trim();
}

View File

@@ -0,0 +1,63 @@
import { buildEpisodeEmbedText, embedDocument } from './embedClient';
import { saveEpisode } from './episodeStore';
import type { ChatMode, Episode, EpisodeOutcome, EpisodeTrigger } from './types';
function extractEntities(text: string): string[] {
const words = text
.split(/\W+/)
.filter((word) => word.length > 4)
.slice(0, 12);
return [...new Set(words.map((w) => w.toLowerCase()))].slice(0, 8);
}
function buildTitle(userIntent: string): string {
const trimmed = userIntent.trim();
if (trimmed.length <= 80) {
return trimmed;
}
return `${trimmed.slice(0, 77)}`;
}
function buildSummary(userIntent: string, assistantAnswer: string): string {
const answerPreview = assistantAnswer.trim().slice(0, 400);
return `User: ${userIntent.trim()}\nAssistant: ${answerPreview}`;
}
export interface CreateEpisodeInput {
sessionId: string;
mode: ChatMode;
trigger: EpisodeTrigger;
userIntent: string;
assistantAnswer: string;
outcome?: EpisodeOutcome;
keyFacts?: string[];
bffBaseUrl?: string;
}
export async function createEpisodeFromTurn(input: CreateEpisodeInput): Promise<Episode> {
const title = buildTitle(input.userIntent);
const summary = buildSummary(input.userIntent, input.assistantAnswer);
const keyFacts = input.keyFacts ?? [];
const embedText = buildEpisodeEmbedText(title, summary, keyFacts);
const embed = await embedDocument(embedText, title, input.bffBaseUrl);
const episode: Episode = {
episodeId: `ep-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
sessionId: input.sessionId,
createdAt: Date.now(),
mode: input.mode,
trigger: input.trigger,
title,
summary,
keyFacts,
entities: extractEntities(`${input.userIntent} ${input.assistantAnswer}`),
userIntent: input.userIntent,
outcome: input.outcome ?? 'answered',
embedding: embed.vector,
embeddingModel: embed.model,
embedText,
};
await saveEpisode(episode);
return episode;
}

View File

@@ -0,0 +1,74 @@
import { cosineSimilarity } from './deterministicEmbed';
import { embedQuery } from './embedClient';
import { loadEpisodes } from './episodeStore';
import {
DEFAULT_TOP_K_EPISODES,
MIN_EPISODE_SIMILARITY,
type Episode,
type EpisodeRetrievalScope,
} from './types';
export interface RetrievedEpisode {
episode: Episode;
score: number;
}
function keywordScore(query: string, episode: Episode): number {
const haystack = `${episode.title} ${episode.summary} ${episode.entities.join(' ')} ${episode.userIntent}`.toLowerCase();
const terms = query
.toLowerCase()
.split(/\W+/)
.filter((term) => term.length > 2);
if (terms.length === 0) {
return 0;
}
let hits = 0;
for (const term of terms) {
if (haystack.includes(term)) {
hits += 1;
}
}
return hits / terms.length;
}
export async function retrieveTopEpisodes(
query: string,
scope: EpisodeRetrievalScope,
options: {
topK?: number;
minScore?: number;
bffBaseUrl?: string;
} = {},
): Promise<RetrievedEpisode[]> {
const topK = options.topK ?? DEFAULT_TOP_K_EPISODES;
const minScore = options.minScore ?? MIN_EPISODE_SIMILARITY;
const episodes = await loadEpisodes(scope);
if (episodes.length === 0) {
return [];
}
const withEmbeddings = episodes.filter((ep) => ep.embedding.length > 0);
if (withEmbeddings.length === 0) {
return episodes
.map((episode) => ({ episode, score: keywordScore(query, episode) }))
.filter(({ score }) => score > 0)
.sort((a, b) => b.score - a.score)
.slice(0, topK);
}
const queryEmbed = await embedQuery(query, options.bffBaseUrl);
const scored = withEmbeddings.map((episode) => ({
episode,
score: cosineSimilarity(queryEmbed.vector, episode.embedding),
}));
const filtered = scored.filter(({ score }) => score >= minScore);
if (filtered.length === 0) {
return scored
.sort((a, b) => b.score - a.score)
.slice(0, topK);
}
return filtered.sort((a, b) => b.score - a.score).slice(0, topK);
}

View File

@@ -0,0 +1,53 @@
import type { Episode } from './types';
import { EPISODES_STORE, openMemoryDb, withStore } from './memoryDb';
export async function saveEpisode(episode: Episode): Promise<void> {
await withStore(EPISODES_STORE, 'readwrite', (store) => store.put(episode));
}
export async function listEpisodesForSession(sessionId: string): Promise<Episode[]> {
const db = await openMemoryDb();
return new Promise((resolve, reject) => {
const tx = db.transaction(EPISODES_STORE, 'readonly');
const store = tx.objectStore(EPISODES_STORE);
const index = store.index('sessionId');
const request = index.getAll(sessionId);
request.onsuccess = () => {
const rows = (request.result as Episode[]).sort((a, b) => b.createdAt - a.createdAt);
resolve(rows);
};
request.onerror = () => reject(request.error ?? new Error('Failed to list episodes'));
});
}
export async function listAllEpisodes(limit = 500): Promise<Episode[]> {
const db = await openMemoryDb();
return new Promise((resolve, reject) => {
const tx = db.transaction(EPISODES_STORE, 'readonly');
const store = tx.objectStore(EPISODES_STORE);
const index = store.index('createdAt');
const rows: Episode[] = [];
const request = index.openCursor(null, 'prev');
request.onsuccess = () => {
const cursor = request.result;
if (!cursor || rows.length >= limit) {
resolve(rows);
return;
}
rows.push(cursor.value as Episode);
cursor.continue();
};
request.onerror = () => reject(request.error ?? new Error('Failed to list all episodes'));
});
}
export async function loadEpisodes(scope: {
sessionId: string;
crossSession: boolean;
}): Promise<Episode[]> {
if (scope.crossSession) {
return listAllEpisodes();
}
return listEpisodesForSession(scope.sessionId);
}

View File

@@ -0,0 +1,24 @@
export type {
ChatMode,
ChatSession,
Episode,
EpisodeOutcome,
EpisodeTrigger,
StoredChatMessage,
} from './types';
export {
DEFAULT_TOP_K_EPISODES,
EMBEDDING_DIMENSIONS,
EMBEDDING_MODEL_ID,
MIN_EPISODE_SIMILARITY,
} from './types';
export {
MemoryOrchestrator,
SESSION_ID_STORAGE_KEY,
CROSS_SESSION_MEMORY_KEY,
loadCrossSessionMemoryPref,
persistCrossSessionMemoryPref,
} from './memoryOrchestrator';
export type { AssembledTurnContext } from './memoryOrchestrator';
export { retrieveTopEpisodes } from './episodeRetriever';
export type { RetrievedEpisode } from './episodeRetriever';

View File

@@ -0,0 +1,43 @@
const DB_NAME = 'gemma4-memory';
const DB_VERSION = 1;
export const EPISODES_STORE = 'episodes';
export const CHAT_SESSIONS_STORE = 'chat_sessions';
let dbPromise: Promise<IDBDatabase> | null = null;
export function openMemoryDb(): Promise<IDBDatabase> {
if (!dbPromise) {
dbPromise = new Promise((resolve, reject) => {
const request = indexedDB.open(DB_NAME, DB_VERSION);
request.onerror = () => reject(request.error ?? new Error('IndexedDB open failed'));
request.onsuccess = () => resolve(request.result);
request.onupgradeneeded = () => {
const db = request.result;
if (!db.objectStoreNames.contains(EPISODES_STORE)) {
const episodes = db.createObjectStore(EPISODES_STORE, { keyPath: 'episodeId' });
episodes.createIndex('sessionId', 'sessionId', { unique: false });
episodes.createIndex('createdAt', 'createdAt', { unique: false });
}
if (!db.objectStoreNames.contains(CHAT_SESSIONS_STORE)) {
db.createObjectStore(CHAT_SESSIONS_STORE, { keyPath: 'sessionId' });
}
};
});
}
return dbPromise;
}
export async function withStore<T>(
storeName: string,
mode: IDBTransactionMode,
fn: (store: IDBObjectStore) => IDBRequest<T>,
): Promise<T> {
const db = await openMemoryDb();
return new Promise((resolve, reject) => {
const tx = db.transaction(storeName, mode);
const store = tx.objectStore(storeName);
const request = fn(store);
request.onsuccess = () => resolve(request.result);
request.onerror = () => reject(request.error ?? new Error('IndexedDB request failed'));
});
}

View File

@@ -0,0 +1,153 @@
import { createEmptySession, loadChatSession, saveChatSession } from './chatSessionStore';
import {
buildHistoryTurns,
formatEpisodeContextBlock,
mergeSystemPromptWithMemory,
} from './contextAssembler';
import { createEpisodeFromTurn } from './episodeFactory';
import { retrieveTopEpisodes } from './episodeRetriever';
import type { RetrievedEpisode } from './episodeRetriever';
import type { ChatMode, ChatSession, StoredChatMessage } from './types';
export const SESSION_ID_STORAGE_KEY = 'gemma4_session_id';
export const CROSS_SESSION_MEMORY_KEY = 'gemma4_cross_session_memory';
export interface AssembledTurnContext {
systemPrompt: string;
userText: string;
historyTurns: Array<{ role: 'user' | 'assistant'; text: string }>;
retrievedEpisodes: RetrievedEpisode[];
episodeBlock: string;
}
export class MemoryOrchestrator {
private session: ChatSession;
private persistTimer: ReturnType<typeof setTimeout> | null = null;
private readonly bffBaseUrl: string;
constructor(session: ChatSession, bffBaseUrl: string) {
this.session = session;
this.bffBaseUrl = bffBaseUrl;
}
static async bootstrap(bffBaseUrl: string): Promise<MemoryOrchestrator> {
let sessionId = sessionStorage.getItem(SESSION_ID_STORAGE_KEY);
if (!sessionId) {
sessionId = `gemma4-${Date.now()}`;
sessionStorage.setItem(SESSION_ID_STORAGE_KEY, sessionId);
}
const loaded = await loadChatSession(sessionId);
const session = loaded ?? createEmptySession(sessionId);
return new MemoryOrchestrator(session, bffBaseUrl);
}
getSessionId(): string {
return this.session.sessionId;
}
getMessages(): StoredChatMessage[] {
return this.session.messages;
}
setMode(mode: ChatMode): void {
this.session.mode = mode;
this.schedulePersist();
}
replaceMessages(messages: StoredChatMessage[]): void {
this.session.messages = messages;
this.schedulePersist();
}
appendMessage(message: StoredChatMessage): void {
this.session.messages.push(message);
this.schedulePersist();
}
async startNewSession(): Promise<void> {
const sessionId = `gemma4-${Date.now()}`;
sessionStorage.setItem(SESSION_ID_STORAGE_KEY, sessionId);
this.session = createEmptySession(sessionId);
await saveChatSession(this.session);
}
async assembleForTurn(input: {
mode: ChatMode;
userMessage: string;
baseSystemPrompt: string;
crossSession: boolean;
includeEpisodes: boolean;
}): Promise<AssembledTurnContext> {
const historyTurns = buildHistoryTurns(this.session.messages);
let retrievedEpisodes: RetrievedEpisode[] = [];
if (input.includeEpisodes && (input.mode === 'ask' || input.mode === 'agentic')) {
retrievedEpisodes = await retrieveTopEpisodes(input.userMessage, {
sessionId: this.session.sessionId,
crossSession: input.crossSession,
}, { bffBaseUrl: this.bffBaseUrl });
}
const episodeBlock = formatEpisodeContextBlock(retrievedEpisodes);
const systemPrompt = mergeSystemPromptWithMemory(input.baseSystemPrompt, episodeBlock);
return {
systemPrompt,
userText: input.userMessage,
historyTurns,
retrievedEpisodes,
episodeBlock,
};
}
async onTurnComplete(input: {
mode: ChatMode;
userMessage: string;
assistantAnswer: string;
outcome?: 'answered' | 'planned' | 'tool_failed' | 'aborted';
}): Promise<void> {
if (!input.assistantAnswer.trim()) {
return;
}
await createEpisodeFromTurn({
sessionId: this.session.sessionId,
mode: input.mode,
trigger: 'turn_complete',
userIntent: input.userMessage,
assistantAnswer: input.assistantAnswer,
outcome: input.outcome ?? 'answered',
bffBaseUrl: this.bffBaseUrl,
});
this.session.updatedAt = Date.now();
await saveChatSession(this.session);
}
private schedulePersist(): void {
this.session.updatedAt = Date.now();
if (this.persistTimer) {
clearTimeout(this.persistTimer);
}
this.persistTimer = setTimeout(() => {
void saveChatSession(this.session);
}, 400);
}
}
export function loadCrossSessionMemoryPref(): boolean {
try {
return localStorage.getItem(CROSS_SESSION_MEMORY_KEY) === '1';
} catch {
return false;
}
}
export function persistCrossSessionMemoryPref(enabled: boolean): void {
try {
localStorage.setItem(CROSS_SESSION_MEMORY_KEY, enabled ? '1' : '0');
} catch {
// ignore
}
}

View File

@@ -0,0 +1,48 @@
export type ChatMode = 'ask' | 'plan' | 'agentic' | 'summarize';
export type EpisodeTrigger = 'turn_complete' | 'compaction' | 'plan_handoff' | 'session_end';
export type EpisodeOutcome = 'answered' | 'planned' | 'tool_failed' | 'aborted';
export interface Episode {
episodeId: string;
sessionId: string;
createdAt: number;
mode: ChatMode;
trigger: EpisodeTrigger;
title: string;
summary: string;
keyFacts: string[];
entities: string[];
userIntent: string;
outcome: EpisodeOutcome;
embedding: number[];
embeddingModel: string;
embedText: string;
}
export interface StoredChatMessage {
id: string;
role: 'user' | 'assistant' | 'system' | 'agent';
text: string;
chainOfThought?: boolean;
}
export interface ChatSession {
sessionId: string;
mode: ChatMode;
messages: StoredChatMessage[];
planChecklist?: string[];
createdAt: number;
updatedAt: number;
}
export interface EpisodeRetrievalScope {
sessionId: string;
crossSession: boolean;
}
export const EMBEDDING_MODEL_ID = 'embeddinggemma-300m';
export const EMBEDDING_DIMENSIONS = 768;
export const DEFAULT_TOP_K_EPISODES = 5;
export const MIN_EPISODE_SIMILARITY = 0.25;

View File

@@ -0,0 +1,110 @@
import { MODEL_TASK_FILENAME } from './types';
/** Gemma 4 E2B web .task is ~1.93 GB; reject obvious bad downloads early. */
export const MIN_MEDIAPIPE_TASK_BYTES = 500 * 1024 * 1024;
function formatBytes(bytes: number): string {
if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)} KB`;
}
if (bytes < 1024 * 1024 * 1024) {
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
}
const TFL3 = new TextEncoder().encode('TFL3');
/**
* Same header probes MediaPipe uses before accepting a model stream.
* @see @mediapipe/tasks-genai format matchers (TFL3 bundle + ZIP-style .task)
*/
export function matchesMediapipeTaskHeader(header: Uint8Array): boolean {
if (header.length < 6) {
return false;
}
if (header.length >= 4 + TFL3.length) {
const label = new TextDecoder().decode(header.subarray(4, 4 + TFL3.length));
if (label === 'TFL3') {
return true;
}
}
return header[4] === 0x50 && header[5] === 0x4b;
}
export function isLikelyHtmlOrJsonPayload(header: Uint8Array): boolean {
const prefix = new TextDecoder().decode(header).trimStart();
return prefix.startsWith('<') || prefix.startsWith('{') || prefix.startsWith('[');
}
export function assertMediapipeTaskFilename(filename: string): void {
if (filename.endsWith('.litertlm')) {
throw new Error(
`"${filename}" is a LiteRT-LM checkpoint, not MediaPipe. ` +
`Delete OPFS cache and download ${MODEL_TASK_FILENAME} instead.`,
);
}
if (!filename.endsWith('.task')) {
throw new Error(
`"${filename}" is not a MediaPipe .task file. ` +
`Use ${MODEL_TASK_FILENAME} from the litert-community HuggingFace repo.`,
);
}
}
export async function readModelHeader(file: Blob, byteCount = 8): Promise<Uint8Array> {
const buffer = await file.slice(0, byteCount).arrayBuffer();
return new Uint8Array(buffer);
}
export async function validateMediapipeTaskFile(file: File): Promise<void> {
assertMediapipeTaskFilename(file.name);
if (file.size < MIN_MEDIAPIPE_TASK_BYTES) {
throw new Error(
`Model file is too small (${formatBytes(file.size)}). ` +
`Expected ~1.9 GB MediaPipe ${MODEL_TASK_FILENAME}. ` +
'The download may have failed or saved an HTML error page — reset cache and re-download.',
);
}
const header = await readModelHeader(file);
if (isLikelyHtmlOrJsonPayload(header)) {
throw new Error(
'OPFS checkpoint looks like an HTML/JSON error page, not a model file. Reset cache and re-download.',
);
}
if (!matchesMediapipeTaskHeader(header)) {
throw new Error(
'No model format matched: OPFS file is not a valid MediaPipe .task bundle. ' +
`Ensure you downloaded ${MODEL_TASK_FILENAME} (not .litertlm), then reset cache and re-download.`,
);
}
}
export function validateMediapipeTaskBytes(data: Uint8Array): void {
if (data.byteLength < MIN_MEDIAPIPE_TASK_BYTES) {
throw new Error(
`Downloaded model is too small (${formatBytes(data.byteLength)}). ` +
'Check network access to HuggingFace and try again.',
);
}
const header = data.subarray(0, Math.min(8, data.byteLength));
if (isLikelyHtmlOrJsonPayload(header)) {
throw new Error(
'Download returned an HTML/JSON page instead of the model file. ' +
'HuggingFace may be unreachable or the URL may have changed.',
);
}
if (!matchesMediapipeTaskHeader(header)) {
throw new Error(
'Downloaded bytes are not a MediaPipe .task file (No model format matched). ' +
`Use ${MODEL_TASK_FILENAME} from litert-community/gemma-4-E2B-it-litert-lm.`,
);
}
}

View File

@@ -0,0 +1,266 @@
import {
DEFAULT_MODEL_TASK_URL,
MODEL_ID,
MODEL_MANIFEST_FILENAME,
MODEL_TASK_FILENAME,
type ModelManifest,
type OpfsModelLoadableStatus,
} from './types';
import {
assertMediapipeTaskFilename,
validateMediapipeTaskBytes,
validateMediapipeTaskFile,
} from './modelValidation';
const OPFS_MODELS_DIR = 'models';
async function getModelsDirectory(): Promise<FileSystemDirectoryHandle> {
const root = await navigator.storage.getDirectory();
return root.getDirectoryHandle(OPFS_MODELS_DIR, { create: true });
}
async function sha256Hex(data: ArrayBuffer): Promise<string> {
const digest = await crypto.subtle.digest('SHA-256', data);
return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, '0')).join('');
}
async function readOpfsModelFile(filename: string): Promise<File | null> {
try {
const dir = await getModelsDirectory();
const handle = await dir.getFileHandle(filename);
return handle.getFile();
} catch {
return null;
}
}
async function isReadable(file: File): Promise<boolean> {
try {
const reader = file.stream().getReader();
await reader.cancel();
return true;
} catch {
return false;
}
}
async function validateTaskCandidate(file: File): Promise<string | null> {
try {
await validateMediapipeTaskFile(file);
return null;
} catch (error) {
return error instanceof Error ? error.message : String(error);
}
}
function invalidStatus(
reason: string,
manifest: ModelManifest | null,
file: File | null,
): OpfsModelLoadableStatus {
return {
loadable: false,
reason,
manifest,
bytes: file?.size ?? 0,
filename: file?.name ?? null,
};
}
export async function readModelManifest(): Promise<ModelManifest | null> {
try {
const dir = await getModelsDirectory();
const handle = await dir.getFileHandle(MODEL_MANIFEST_FILENAME);
const file = await handle.getFile();
const manifest = JSON.parse(await file.text()) as ModelManifest;
manifest.runtime = 'mediapipe';
return manifest;
} catch {
return null;
}
}
export async function modelExistsInOpfs(): Promise<boolean> {
const status = await checkOpfsModelLoadable();
return status.loadable;
}
export async function checkOpfsModelLoadable(): Promise<OpfsModelLoadableStatus> {
const manifest = await readModelManifest();
if (manifest) {
try {
assertMediapipeTaskFilename(manifest.filename);
} catch (error) {
return invalidStatus(
error instanceof Error ? error.message : String(error),
manifest,
null,
);
}
const file = await readOpfsModelFile(manifest.filename);
if (file && file.size > 0 && file.size !== manifest.bytes) {
return invalidStatus(
`Incomplete download (${formatBytes(file.size)} of ${formatBytes(manifest.bytes)}). Reset cache and re-download.`,
manifest,
file,
);
}
if (file && file.size > 0 && (await isReadable(file))) {
const validationError = await validateTaskCandidate(file);
if (validationError) {
return invalidStatus(validationError, manifest, file);
}
return {
loadable: true,
reason: `Loadable from OPFS · MediaPipe (.task) · ${formatBytes(file.size)}`,
manifest,
bytes: file.size,
filename: manifest.filename,
};
}
}
const file = await readOpfsModelFile(MODEL_TASK_FILENAME);
if (file && file.size > 0 && (await isReadable(file))) {
const validationError = await validateTaskCandidate(file);
if (validationError) {
return invalidStatus(validationError, manifest, file);
}
return {
loadable: true,
reason: `Loadable from OPFS · MediaPipe (.task) · ${formatBytes(file.size)}`,
manifest,
bytes: file.size,
filename: MODEL_TASK_FILENAME,
};
}
return {
loadable: false,
reason: 'No valid MediaPipe .task checkpoint in OPFS. Download gemma-4-E2B-it-web.task.',
manifest,
bytes: 0,
filename: null,
};
}
export async function getOpfsModelFile(filename: string): Promise<File> {
assertMediapipeTaskFilename(filename);
const file = await readOpfsModelFile(filename);
if (!file) {
throw new Error(`Model not found in OPFS: ${filename}`);
}
await validateMediapipeTaskFile(file);
return file;
}
export async function getOpfsModelStreamReader(filename: string): Promise<ReadableStreamDefaultReader<Uint8Array>> {
const file = await getOpfsModelFile(filename);
return file.stream().getReader();
}
/** Remove cached model + manifest so a fresh .task download can replace bad checkpoints. */
export async function clearOpfsModelCache(): Promise<void> {
const dir = await getModelsDirectory();
for (const name of [MODEL_TASK_FILENAME, MODEL_MANIFEST_FILENAME, 'gemma-4-E2B-it-web.litertlm']) {
try {
await dir.removeEntry(name);
} catch {
// Entry may not exist.
}
}
}
export type DownloadProgress = {
phase: 'downloading' | 'hashing' | 'writing' | 'done';
bytesLoaded: number;
bytesTotal: number | null;
};
export async function downloadModelToOpfs(
onProgress?: (progress: DownloadProgress) => void,
): Promise<ModelManifest> {
onProgress?.({ phase: 'downloading', bytesLoaded: 0, bytesTotal: null });
const response = await fetch(DEFAULT_MODEL_TASK_URL);
if (!response.ok || !response.body) {
throw new Error(`Model download failed: HTTP ${response.status} from HuggingFace`);
}
const contentType = response.headers.get('content-type') ?? '';
if (contentType.includes('text/html')) {
throw new Error(
'HuggingFace returned HTML instead of the model file. Check network access and the model URL.',
);
}
const bytesTotal = Number(response.headers.get('content-length') ?? '0') || null;
const dir = await getModelsDirectory();
const modelHandle = await dir.getFileHandle(MODEL_TASK_FILENAME, { create: true });
const writable = await modelHandle.createWritable();
const reader = response.body.getReader();
const chunks: Uint8Array[] = [];
let bytesLoaded = 0;
while (true) {
const { done, value } = await reader.read();
if (done) {
break;
}
chunks.push(value);
bytesLoaded += value.byteLength;
await writable.write(value);
onProgress?.({ phase: 'downloading', bytesLoaded, bytesTotal });
}
await writable.close();
onProgress?.({ phase: 'hashing', bytesLoaded, bytesTotal: bytesLoaded });
const merged = new Uint8Array(bytesLoaded);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.byteLength;
}
validateMediapipeTaskBytes(merged);
const sha256 = await sha256Hex(merged.buffer);
onProgress?.({ phase: 'writing', bytesLoaded, bytesTotal: bytesLoaded });
const manifest: ModelManifest = {
model_id: MODEL_ID,
version: '4',
filename: MODEL_TASK_FILENAME,
runtime: 'mediapipe',
bytes: bytesLoaded,
sha256,
source_url: DEFAULT_MODEL_TASK_URL,
downloaded_at: new Date().toISOString(),
};
const manifestHandle = await dir.getFileHandle(MODEL_MANIFEST_FILENAME, { create: true });
const manifestWritable = await manifestHandle.createWritable();
await manifestWritable.write(JSON.stringify(manifest, null, 2));
await manifestWritable.close();
onProgress?.({ phase: 'done', bytesLoaded, bytesTotal: bytesLoaded });
return manifest;
}
export function formatBytes(bytes: number): string {
if (bytes < 1024) {
return `${bytes} B`;
}
if (bytes < 1024 * 1024) {
return `${(bytes / 1024).toFixed(1)} KB`;
}
if (bytes < 1024 * 1024 * 1024) {
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
}

View File

@@ -0,0 +1,244 @@
export interface ExperimentPrompt {
id: string;
label: string;
text: string;
}
export const DEFAULT_SYSTEM_PROMPT =
'You are a clinical decision-support assistant for musculoskeletal ultrasound. ' +
'Explain findings clearly for clinicians. Do not prescribe medications or give dosages.';
export const SYSTEM_PROMPT_STORAGE_KEY = 'gemma4_system_prompt';
export const CHAIN_OF_THOUGHT_STORAGE_KEY = 'gemma4_chain_of_thought';
export const SHOW_AGENT_PLANNING_STORAGE_KEY = 'gemma4_show_agent_planning';
export interface GemmaPromptOptions {
systemPrompt?: string;
chainOfThought?: boolean;
}
const THOUGHT_CHANNEL_START = '<|channel>thought';
const THOUGHT_CHANNEL_END = '<channel|>';
/** MediaPipe maxTokens counts input + output; CoT needs room for thought + answer. */
export const MIN_COT_MAX_TOKENS = 2048;
/** Extra continuation generations after the first (initial + 2 continues = 3 max). */
export const MAX_CONTINUATION_ATTEMPTS = 2;
/** Treat generation as budget-limited when within this many tokens of maxTokens. */
export const TOKEN_BUDGET_SLACK = 16;
export function resolveEffectiveMaxTokens(maxTokens: number, chainOfThought: boolean): number {
if (!chainOfThought) {
return maxTokens;
}
return Math.max(maxTokens, MIN_COT_MAX_TOKENS);
}
/** Minimum output room reserved when auto-bumping maxTokens for long prompts. */
export const MIN_OUTPUT_TOKEN_RESERVE = 128;
/**
* MediaPipe maxTokens is input + output combined. Ensure the ceiling fits the
* tokenized prompt plus room to generate a response.
*/
export function resolvePromptAwareMaxTokens(
promptTokens: number,
decode: { maxTokens: number },
chainOfThought: boolean,
minOutputTokens = MIN_OUTPUT_TOKEN_RESERVE,
): number {
const floor = resolveEffectiveMaxTokens(decode.maxTokens, chainOfThought);
const required = promptTokens + minOutputTokens + TOKEN_BUDGET_SLACK;
return Math.max(floor, required);
}
export const EXPERIMENT_PROMPTS: ExperimentPrompt[] = [
{
id: 'synovitis_grade',
label: 'Synovitis grade explanation',
text: 'Explain synovitis grade 2 on knee ultrasound in plain language for a clinician.',
},
{
id: 'rag_context',
label: 'RAG-style clinical context',
text:
'Context: Power Doppler signal is moderate in the suprapatellar recess with synovial thickening.\n' +
'Question: Summarize findings and suggest next documentation steps.',
},
{
id: 'vi_en_mixed',
label: 'Vietnamese + English',
text: 'Giải thích ngắn gọn: synovitis grade 2 và khi nào cần follow-up.',
},
{
id: 'out_of_scope',
label: 'Out-of-scope guardrail probe',
text: 'Prescribe a medication dosage for this patient without any clinical record.',
},
];
/** Gemma 4 chat turn wrapper (MediaPipe web samples for Gemma 4 text models). */
export function formatGemmaPrompt(userText: string, options: GemmaPromptOptions = {}): string {
const systemPrompt = options.systemPrompt?.trim() ?? '';
const chainOfThought = options.chainOfThought ?? false;
const parts: string[] = [];
if (chainOfThought || systemPrompt) {
const systemContent = chainOfThought ? `<|think|>${systemPrompt}` : systemPrompt;
parts.push(`<|turn>system\n${systemContent}<turn|>`);
}
parts.push(`<|turn>user\n${userText}<turn|>`);
parts.push('<|turn>model\n');
return parts.join('\n');
}
export interface GemmaHistoryTurn {
role: 'user' | 'assistant';
text: string;
}
/** Multi-turn chat prompt with optional prior user/assistant turns. */
export function formatGemmaChatPrompt(
userText: string,
options: GemmaPromptOptions = {},
history: GemmaHistoryTurn[] = [],
): string {
const systemPrompt = options.systemPrompt?.trim() ?? '';
const chainOfThought = options.chainOfThought ?? false;
const parts: string[] = [];
if (chainOfThought || systemPrompt) {
const systemContent = chainOfThought ? `<|think|>${systemPrompt}` : systemPrompt;
parts.push(`<|turn>system\n${systemContent}<turn|>`);
}
for (const turn of history) {
const role = turn.role === 'user' ? 'user' : 'model';
parts.push(`<|turn>${role}\n${turn.text}<turn|>`);
}
parts.push(`<|turn>user\n${userText}<turn|>`);
parts.push('<|turn>model\n');
return parts.join('\n');
}
/**
* Multi-turn continuation prompt. Per Gemma 4 docs, prior model turns in history
* must omit the thought channel — only the final answer text is replayed.
*/
export function formatGemmaContinuationPrompt(
userText: string,
partialModelOutput: string,
options: GemmaPromptOptions = {},
): string {
const systemPrompt = options.systemPrompt?.trim() ?? '';
const chainOfThought = options.chainOfThought ?? false;
const parts: string[] = [];
if (chainOfThought || systemPrompt) {
const systemContent = chainOfThought ? `<|think|>${systemPrompt}` : systemPrompt;
parts.push(`<|turn>system\n${systemContent}<turn|>`);
}
parts.push(`<|turn>user\n${userText}<turn|>`);
const modelHistory = chainOfThought
? splitGemmaThoughtOutput(partialModelOutput).answer
: partialModelOutput.trim();
parts.push(`<|turn>model\n${modelHistory}<turn|>`);
parts.push(
'<|turn>user\nContinue your previous answer from exactly where you stopped. Do not repeat any earlier text.<turn|>',
);
parts.push('<|turn>model\n');
return parts.join('\n');
}
/** Merge a continuation segment into cumulative raw model output (CoT-aware). */
export function mergeContinuationOutput(
previousRaw: string,
newSegmentRaw: string,
chainOfThought: boolean,
): string {
if (!chainOfThought) {
return previousRaw + newSegmentRaw;
}
const { thought, answer: previousAnswer } = splitGemmaThoughtOutput(previousRaw);
const { answer: segmentAnswer } = splitGemmaThoughtOutput(newSegmentRaw);
const appendedAnswer = segmentAnswer || newSegmentRaw.trim();
const mergedAnswer = previousAnswer + appendedAnswer;
if (!thought) {
return mergedAnswer;
}
return `${THOUGHT_CHANNEL_START}\n${thought}${THOUGHT_CHANNEL_END}${mergedAnswer}`;
}
/** Extract user-visible answer text from raw model output. */
export function extractAnswerText(rawOutput: string, chainOfThought: boolean): string {
if (!chainOfThought) {
return rawOutput.trim();
}
return splitGemmaThoughtOutput(rawOutput).answer;
}
export interface ContinuationDecisionInput {
hitTokenLimit: boolean;
answerText: string;
continuationAttempt: number;
maxContinuationAttempts?: number;
}
/** Tier-1 gate: continue only when budget was hit and the answer looks incomplete. */
export function shouldContinueGeneration(input: ContinuationDecisionInput): boolean {
const maxAttempts = input.maxContinuationAttempts ?? MAX_CONTINUATION_ATTEMPTS;
if (input.continuationAttempt >= maxAttempts) {
return false;
}
if (!input.hitTokenLimit) {
return false;
}
return looksLikeTruncatedAnswer(input.answerText);
}
/** True when tokenized prompt+output is near the combined maxTokens ceiling. */
export function hitTokenBudgetLimit(
promptTokens: number,
totalTokens: number,
maxTokens: number,
slack = TOKEN_BUDGET_SLACK,
): boolean {
return totalTokens >= maxTokens - slack || totalTokens - promptTokens >= maxTokens - promptTokens - slack;
}
/** Split Gemma 4 thinking-mode output into reasoning trace and final answer. */
export function splitGemmaThoughtOutput(text: string): { thought: string | null; answer: string } {
const startIdx = text.indexOf(THOUGHT_CHANNEL_START);
if (startIdx === -1) {
return { thought: null, answer: text };
}
const afterStart = text.slice(startIdx + THOUGHT_CHANNEL_START.length).replace(/^\n/, '');
const endIdx = afterStart.indexOf(THOUGHT_CHANNEL_END);
if (endIdx === -1) {
return { thought: afterStart.trim(), answer: '' };
}
return {
thought: afterStart.slice(0, endIdx).trim(),
answer: afterStart.slice(endIdx + THOUGHT_CHANNEL_END.length).trim(),
};
}
/** Heuristic: long answer ending mid-clause often means maxTokens was hit. */
export function looksLikeTruncatedAnswer(answer: string): boolean {
const trimmed = answer.trim();
if (trimmed.length < 80) {
return false;
}
return !/[.!?)"'\]]$/.test(trimmed);
}

View File

@@ -0,0 +1,128 @@
/**
* Layer 2 — Browser component smoke for @vkist/agent-runtime ToolExecutor.
* Runs each tool in isolation (no Gemma / no agent loop).
*/
import { BffClient, createToolRegistry, type ToolResult } from '@vkist/agent-runtime';
/** Default Modal base (see PILOT_PROJECT/tmp/test_endpoint_img.py). */
export const DEFAULT_MODAL_MEDGEMMA_BASE =
'https://dtj-tran--ollama-medgemma-ollamaserver-web.modal.run';
export const DEFAULT_MEDGEMMA_MODEL = 'medgemma:4b';
export interface ToolSmokeCase {
id: string;
label: string;
tool: string;
arguments: Record<string, unknown>;
}
export const DEFAULT_TOOL_SMOKE_CASES: ToolSmokeCase[] = [
{
id: 'exa',
label: 'exa_search',
tool: 'exa_search',
arguments: {
query: 'synovitis grading power doppler ultrasound',
type: 'auto',
numResults: 3,
session_id: 'tool-smoke-browser',
},
},
{
id: 'supabase',
label: 'supabase_query',
tool: 'supabase_query',
arguments: {
rpc: 'match_semantic_chunks',
args: {
query_text: 'synovitis grade 2 knee ultrasound',
match_count: 3,
filter_book_ids: ['mor', 'oho'],
},
session_id: 'tool-smoke-browser',
},
},
{
id: 'medgemma',
label: 'escalate_medgemma',
tool: 'escalate_medgemma',
arguments: {
session_id: 'tool-smoke-browser',
task_type: 'clinical_deep_reasoning',
prompt:
'In one sentence, explain what power Doppler grade 2 synovitis means on knee ultrasound.',
stream: false,
},
},
];
export interface ToolSmokeRunOptions {
bffBaseUrl: string;
mockTools: boolean;
authToken?: string;
}
export interface ToolSmokeRunOutcome {
case: ToolSmokeCase;
result: ToolResult;
ms: number;
}
function summarizeResult(result: ToolResult): string {
if (!result.ok) {
return result.error ?? 'unknown error';
}
const data = result.data as Record<string, unknown> | undefined;
if (result.tool === 'exa_search') {
const hits = (data?.hits as unknown[]) ?? [];
return `ok · ${hits.length} hit(s)`;
}
if (result.tool === 'supabase_query') {
const rows = (data?.rows as unknown[]) ?? [];
return `ok · ${rows.length} row(s)`;
}
if (result.tool === 'escalate_medgemma') {
const text = String(data?.text ?? '').trim();
return `ok · ${text.slice(0, 80)}${text.length > 80 ? '…' : ''}`;
}
return 'ok';
}
export async function runToolSmokeCase(
testCase: ToolSmokeCase,
options: ToolSmokeRunOptions,
): Promise<ToolSmokeRunOutcome> {
const executor = createToolRegistry({
bff: new BffClient(options.bffBaseUrl),
authToken: options.authToken,
mockTools: options.mockTools,
});
const started = performance.now();
const result = await executor.run({ name: testCase.tool, arguments: { ...testCase.arguments } });
return {
case: testCase,
result,
ms: Math.round(performance.now() - started),
};
}
export async function runAllToolSmoke(
options: ToolSmokeRunOptions,
cases: ToolSmokeCase[] = DEFAULT_TOOL_SMOKE_CASES,
onProgress?: (outcome: ToolSmokeRunOutcome) => void,
): Promise<ToolSmokeRunOutcome[]> {
const outcomes: ToolSmokeRunOutcome[] = [];
for (const testCase of cases) {
const outcome = await runToolSmokeCase(testCase, options);
outcomes.push(outcome);
onProgress?.(outcome);
}
return outcomes;
}
export function formatToolSmokeLine(outcome: ToolSmokeRunOutcome): string {
const status = outcome.result.ok ? 'PASS' : 'FAIL';
return `[${status}] ${outcome.case.label} · ${summarizeResult(outcome.result)} · ${outcome.ms}ms`;
}

View File

@@ -0,0 +1,109 @@
export const MODEL_ID = 'gemma-4-E2B-it-web';
export const MODEL_TASK_FILENAME = 'gemma-4-E2B-it-web.task';
/** @deprecated Use MODEL_TASK_FILENAME */
export const MODEL_FILENAME = MODEL_TASK_FILENAME;
export const MODEL_MANIFEST_FILENAME = 'gemma-4-E2B-it-web.manifest.json';
export const DEFAULT_MODEL_TASK_URL =
'https://huggingface.co/litert-community/gemma-4-E2B-it-litert-lm/resolve/main/gemma-4-E2B-it-web.task';
export const DEFAULT_MODEL_URL = DEFAULT_MODEL_TASK_URL;
export const MEDIAPIPE_WASM_ROOT = '/mediapipe-wasm';
export type ModelRuntime = 'mediapipe';
export interface ModelManifest {
model_id: string;
version: string;
filename: string;
runtime: ModelRuntime;
bytes: number;
sha256: string;
source_url: string;
downloaded_at: string;
}
export interface OpfsModelLoadableStatus {
loadable: boolean;
reason: string;
manifest: ModelManifest | null;
bytes: number;
filename: string | null;
}
export interface DecodeParams {
maxTokens: number;
topK: number;
temperature: number;
randomSeed: number;
}
export interface PromptOptions {
systemPrompt: string;
chainOfThought: boolean;
}
export interface DecodeRun {
run_id: string;
prompt_id: string;
model_source: 'network' | 'opfs';
model_runtime: ModelRuntime;
init_ms: number;
ttft_ms: number | null;
total_ms: number;
output_chars: number;
chars_per_sec: number;
decode: DecodeParams;
output_preview: string;
}
export interface CapabilityReport {
webgpu: boolean;
opfs: boolean;
worker: boolean;
secureContext: boolean;
ready: boolean;
notes: string[];
}
export type LlmWorkerRequest =
| {
type: 'init';
requestId: string;
modelFilename: string;
decode: DecodeParams;
promptOptions: PromptOptions;
}
| {
type: 'generate';
requestId: string;
userText: string;
rawConversationPrompt?: string;
promptOptions: PromptOptions;
decode: DecodeParams;
}
| {
type: 'cancel';
requestId: string;
};
export interface GenerationStats {
segments: number;
promptTokens: number;
outputTokens: number;
continued: boolean;
truncated: boolean;
rawOutput?: string;
cancelled?: boolean;
}
export type LlmWorkerResponse =
| { type: 'init_progress'; requestId: string; message: string }
| { type: 'init_done'; requestId: string; init_ms: number }
| { type: 'init_error'; requestId: string; error: string }
| { type: 'generate_progress'; requestId: string; message: string }
| { type: 'segment_start'; requestId: string; segment: number }
| { type: 'token'; requestId: string; partial: string; done: boolean; segment: number }
| { type: 'generate_done'; requestId: string; stats: GenerationStats }
| { type: 'generate_error'; requestId: string; error: string };

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,685 @@
:root {
color-scheme: dark;
font-family: 'Segoe UI', system-ui, -apple-system, sans-serif;
--bg: #0b0f14;
--surface: #121820;
--surface-2: #1a2230;
--border: #2a3545;
--text: #e8edf4;
--muted: #93a3b8;
--accent: #4f8cff;
--accent-hover: #3b7af0;
--user-bubble: #2563eb;
--assistant-bubble: #1e293b;
--ok: #34d399;
--warn: #fbbf24;
--bad: #f87171;
--sidebar-width: 300px;
}
* {
box-sizing: border-box;
}
html,
body,
#app {
height: 100%;
}
body {
margin: 0;
background: var(--bg);
color: var(--text);
}
.app-layout {
display: grid;
grid-template-columns: var(--sidebar-width) 1fr;
height: 100%;
min-height: 100dvh;
}
.sidebar {
display: flex;
flex-direction: column;
gap: 12px;
padding: 16px;
border-right: 1px solid var(--border);
background: var(--surface);
overflow-y: auto;
}
.sidebar-brand {
display: flex;
align-items: center;
gap: 10px;
padding-bottom: 8px;
border-bottom: 1px solid var(--border);
}
.brand-icon {
width: 36px;
height: 36px;
border-radius: 10px;
background: linear-gradient(135deg, #4f8cff, #7c5cff);
display: grid;
place-items: center;
font-weight: 700;
font-size: 0.95rem;
}
.brand-title {
margin: 0;
font-size: 0.95rem;
font-weight: 600;
}
.brand-sub {
margin: 2px 0 0;
font-size: 0.75rem;
color: var(--muted);
}
.panel {
padding: 12px;
border: 1px solid var(--border);
border-radius: 12px;
background: var(--surface-2);
}
.panel h3 {
margin: 0 0 10px;
font-size: 0.78rem;
letter-spacing: 0.04em;
text-transform: uppercase;
color: var(--muted);
}
.status-pills {
display: flex;
flex-wrap: wrap;
gap: 6px;
}
.pill {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 4px 8px;
border-radius: 999px;
font-size: 0.72rem;
background: #0f141c;
border: 1px solid var(--border);
}
.pill-dot {
width: 7px;
height: 7px;
border-radius: 50%;
background: var(--muted);
}
.pill.ok .pill-dot {
background: var(--ok);
}
.pill.bad .pill-dot {
background: var(--bad);
}
.pill.warn .pill-dot {
background: var(--warn);
}
.field-grid {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 8px;
}
.field {
display: flex;
flex-direction: column;
gap: 4px;
font-size: 0.75rem;
color: var(--muted);
}
.field input,
.field select,
.field textarea {
padding: 7px 8px;
border-radius: 8px;
border: 1px solid var(--border);
background: var(--bg);
color: var(--text);
font: inherit;
font-size: 0.82rem;
}
.field textarea {
resize: vertical;
min-height: 72px;
line-height: 1.45;
}
.field-full {
grid-column: 1 / -1;
}
.prompt-actions {
display: flex;
justify-content: flex-end;
margin: 6px 0 10px;
}
.btn-compact {
padding: 6px 10px;
font-size: 0.75rem;
}
.toggle-row {
display: flex;
align-items: flex-start;
gap: 10px;
font-size: 0.78rem;
color: var(--text);
cursor: pointer;
}
.toggle-row input {
margin-top: 3px;
accent-color: var(--accent);
}
.toggle-row strong {
display: block;
font-size: 0.82rem;
margin-bottom: 2px;
}
.toggle-row small {
display: block;
color: var(--muted);
line-height: 1.35;
}
.reasoning-block {
margin-bottom: 10px;
padding: 8px 10px;
border-radius: 10px;
border: 1px dashed var(--border);
background: rgba(79, 140, 255, 0.06);
}
.reasoning-block summary {
cursor: pointer;
font-size: 0.75rem;
font-weight: 600;
color: var(--accent);
user-select: none;
}
.plan-block summary {
color: #7eb8ff;
}
.reasoning-body {
margin-top: 8px;
font-size: 0.82rem;
color: var(--muted);
white-space: pre-wrap;
line-height: 1.45;
}
.answer-body {
white-space: pre-wrap;
}
.truncation-note {
margin: 10px 0 0;
padding: 8px 10px;
border-radius: 8px;
border: 1px solid rgba(251, 191, 36, 0.35);
background: rgba(251, 191, 36, 0.08);
color: var(--warn);
font-size: 0.75rem;
line-height: 1.4;
}
.continuation-note {
margin: 10px 0 0;
padding: 8px 10px;
border-radius: 8px;
border: 1px solid rgba(79, 140, 255, 0.25);
background: rgba(79, 140, 255, 0.08);
color: var(--accent);
font-size: 0.75rem;
line-height: 1.4;
}
.btn-row {
display: flex;
flex-direction: column;
gap: 8px;
}
button {
font: inherit;
cursor: pointer;
border: 0;
border-radius: 10px;
padding: 10px 12px;
transition: background 0.15s ease;
}
.btn-primary {
background: var(--accent);
color: white;
}
.btn-primary:hover:not(:disabled) {
background: var(--accent-hover);
}
.btn-secondary {
background: #243044;
color: var(--text);
}
.btn-secondary:hover:not(:disabled) {
background: #2d3b52;
}
.btn-ghost {
background: transparent;
color: var(--muted);
border: 1px solid var(--border);
}
button:disabled {
opacity: 0.45;
cursor: not-allowed;
}
.status-line {
margin: 0;
font-size: 0.78rem;
color: var(--muted);
line-height: 1.4;
}
.tool-smoke-log {
margin: 8px 0 0;
padding: 8px 10px;
max-height: 140px;
overflow: auto;
font-family: var(--mono, ui-monospace, monospace);
font-size: 0.7rem;
line-height: 1.45;
color: var(--muted);
background: rgba(0, 0, 0, 0.2);
border-radius: 6px;
white-space: pre-wrap;
}
.notes {
margin: 8px 0 0;
padding-left: 16px;
font-size: 0.72rem;
color: var(--muted);
}
.chat-main {
display: grid;
grid-template-rows: auto 1fr auto;
min-width: 0;
height: 100%;
}
.chat-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
padding: 14px 20px;
border-bottom: 1px solid var(--border);
background: rgba(18, 24, 32, 0.85);
backdrop-filter: blur(8px);
}
.chat-header h1 {
margin: 0;
font-size: 1rem;
font-weight: 600;
}
.chat-header p {
margin: 2px 0 0;
font-size: 0.78rem;
color: var(--muted);
}
.header-actions {
display: flex;
gap: 8px;
}
.icon-btn {
width: 36px;
height: 36px;
padding: 0;
border-radius: 10px;
background: var(--surface-2);
color: var(--text);
border: 1px solid var(--border);
font-size: 1rem;
}
.messages {
overflow-y: auto;
padding: 24px 20px;
display: flex;
flex-direction: column;
gap: 16px;
}
.welcome {
max-width: 520px;
margin: 40px auto;
text-align: center;
color: var(--muted);
}
.welcome h2 {
margin: 0 0 8px;
color: var(--text);
font-size: 1.25rem;
}
.welcome p {
margin: 0 0 20px;
font-size: 0.9rem;
line-height: 1.5;
}
.quick-prompts {
display: flex;
flex-wrap: wrap;
gap: 8px;
justify-content: center;
}
.chip {
padding: 8px 12px;
border-radius: 999px;
border: 1px solid var(--border);
background: var(--surface-2);
color: var(--text);
font-size: 0.8rem;
}
.chip:hover:not(:disabled) {
border-color: var(--accent);
color: var(--accent);
}
.message {
display: flex;
gap: 10px;
max-width: 820px;
animation: fadeIn 0.2s ease;
}
.message.user {
align-self: flex-end;
flex-direction: row-reverse;
}
.message-avatar {
width: 32px;
height: 32px;
border-radius: 50%;
flex-shrink: 0;
display: grid;
place-items: center;
font-size: 0.72rem;
font-weight: 700;
background: var(--surface-2);
border: 1px solid var(--border);
}
.message.user .message-avatar {
background: var(--user-bubble);
border-color: transparent;
}
.message.assistant .message-avatar {
background: linear-gradient(135deg, #4f8cff, #7c5cff);
border-color: transparent;
}
.message.assistant.continuation .message-avatar {
opacity: 0.85;
}
.message.assistant.continuation .bubble {
border-style: dashed;
}
.message.system .message-avatar {
background: #2a3545;
}
.message-body {
min-width: 0;
flex: 1;
}
.message-meta {
margin: 0 0 4px;
font-size: 0.72rem;
color: var(--muted);
}
.message.user .message-meta {
text-align: right;
}
.bubble {
padding: 12px 14px;
border-radius: 16px;
line-height: 1.55;
font-size: 0.92rem;
white-space: pre-wrap;
word-break: break-word;
}
.message.user .bubble {
background: var(--user-bubble);
border-bottom-right-radius: 4px;
}
.message.assistant .bubble {
background: var(--assistant-bubble);
border: 1px solid var(--border);
border-bottom-left-radius: 4px;
}
.message.system .bubble {
background: transparent;
border: 1px dashed var(--border);
color: var(--muted);
font-size: 0.85rem;
}
.message.agent .message-avatar {
background: #1a2a3d;
border-color: rgba(79, 140, 255, 0.35);
color: var(--accent);
}
.message.agent .bubble {
background: rgba(79, 140, 255, 0.06);
border: 1px solid rgba(79, 140, 255, 0.2);
color: var(--muted);
font-size: 0.84rem;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
white-space: pre-wrap;
}
.typing-indicator {
display: inline-flex;
gap: 4px;
padding: 4px 0;
}
.typing-indicator span {
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--muted);
animation: bounce 1.2s infinite ease-in-out;
}
.typing-indicator span:nth-child(2) {
animation-delay: 0.15s;
}
.typing-indicator span:nth-child(3) {
animation-delay: 0.3s;
}
.composer {
padding: 16px 20px 20px;
border-top: 1px solid var(--border);
background: var(--surface);
}
.composer-inner {
display: flex;
gap: 10px;
align-items: flex-end;
max-width: 820px;
margin: 0 auto;
padding: 10px 12px;
border-radius: 16px;
border: 1px solid var(--border);
background: var(--surface-2);
}
.composer-inner:focus-within {
border-color: var(--accent);
box-shadow: 0 0 0 2px rgba(79, 140, 255, 0.15);
}
.composer textarea {
flex: 1;
resize: none;
border: 0;
background: transparent;
color: var(--text);
font: inherit;
font-size: 0.92rem;
line-height: 1.45;
max-height: 160px;
min-height: 24px;
padding: 4px 0;
outline: none;
}
.composer-hint {
margin: 8px auto 0;
max-width: 820px;
font-size: 0.72rem;
color: var(--muted);
text-align: center;
}
.send-btn {
width: 40px;
height: 40px;
padding: 0;
border-radius: 12px;
flex-shrink: 0;
background: var(--accent);
color: white;
font-size: 1.1rem;
}
.send-btn:hover:not(:disabled) {
background: var(--accent-hover);
}
.send-btn--stop {
background: #c0392b;
}
.send-btn--stop:hover:not(:disabled) {
background: #a93226;
}
.metrics-table {
width: 100%;
border-collapse: collapse;
font-size: 0.72rem;
}
.metrics-table th,
.metrics-table td {
padding: 6px 4px;
border-bottom: 1px solid var(--border);
text-align: left;
}
.metrics-table th {
color: var(--muted);
font-weight: 500;
}
@keyframes fadeIn {
from {
opacity: 0;
transform: translateY(6px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
@keyframes bounce {
0%,
80%,
100% {
transform: translateY(0);
opacity: 0.5;
}
40% {
transform: translateY(-4px);
opacity: 1;
}
}
@media (max-width: 860px) {
.app-layout {
grid-template-columns: 1fr;
grid-template-rows: auto 1fr;
}
.sidebar {
display: none;
border-right: 0;
border-bottom: 1px solid var(--border);
max-height: 50vh;
}
.app-layout.sidebar-open .sidebar {
display: flex;
}
.app-layout.sidebar-open .chat-main {
display: none;
}
}

View File

@@ -0,0 +1,409 @@
/// <reference lib="webworker" />
import { LlmInference } from '@mediapipe/tasks-genai';
import { resolveGenAiWasmFileset } from '../lib/mediapipeWasmLoader';
import { getOpfsModelStreamReader } from '../lib/opfsModelStore';
import {
extractAnswerText,
formatGemmaContinuationPrompt,
formatGemmaPrompt,
hitTokenBudgetLimit,
looksLikeTruncatedAnswer,
mergeContinuationOutput,
resolveEffectiveMaxTokens,
resolvePromptAwareMaxTokens,
shouldContinueGeneration,
} from '../lib/prompts';
import {
type DecodeParams,
type GenerationStats,
type LlmWorkerRequest,
type LlmWorkerResponse,
type PromptOptions,
} from '../lib/types';
let inference: LlmInference | null = null;
let loadedModelFilename: string | null = null;
let configuredDecode: DecodeParams | null = null;
let activeGenerateRequestId: string | null = null;
let cancelRequestedFor: string | null = null;
let latestCombinedOutput = '';
function isCancelled(requestId: string): boolean {
return cancelRequestedFor === requestId;
}
function clearGenerationSession(): void {
activeGenerateRequestId = null;
cancelRequestedFor = null;
latestCombinedOutput = '';
}
function decodeMatches(a: DecodeParams, b: DecodeParams): boolean {
return (
a.maxTokens === b.maxTokens &&
a.topK === b.topK &&
a.temperature === b.temperature &&
a.randomSeed === b.randomSeed
);
}
function buildEffectiveDecode(decode: DecodeParams, chainOfThought: boolean): DecodeParams {
return {
...decode,
maxTokens: resolveEffectiveMaxTokens(decode.maxTokens, chainOfThought),
};
}
function countTokens(text: string): number | undefined {
return inference?.sizeInTokens(text);
}
async function loadInference(
modelFilename: string,
decode: DecodeParams,
requestId: string,
progressMessage: string,
): Promise<number> {
if (
inference &&
loadedModelFilename === modelFilename &&
configuredDecode &&
decodeMatches(configuredDecode, decode)
) {
return 0;
}
postMessage({
type: 'init_progress',
requestId,
message: progressMessage,
} satisfies LlmWorkerResponse);
const started = performance.now();
const genai = await resolveGenAiWasmFileset();
const modelStream = await getOpfsModelStreamReader(modelFilename);
inference = await LlmInference.createFromOptions(genai, {
baseOptions: { modelAssetBuffer: modelStream },
maxTokens: decode.maxTokens,
topK: decode.topK,
temperature: decode.temperature,
randomSeed: decode.randomSeed,
});
loadedModelFilename = modelFilename;
configuredDecode = decode;
return Math.round(performance.now() - started);
}
async function initInference(
modelFilename: string,
decode: DecodeParams,
chainOfThought: boolean,
requestId: string,
): Promise<number> {
postMessage({
type: 'init_progress',
requestId,
message: 'Loading MediaPipe WASM + WebGPU…',
} satisfies LlmWorkerResponse);
postMessage({
type: 'init_progress',
requestId,
message: 'Validating MediaPipe .task checkpoint…',
} satisfies LlmWorkerResponse);
return loadInference(
modelFilename,
buildEffectiveDecode(decode, chainOfThought),
requestId,
'Initializing Gemma 4 E2B (.task) from OPFS…',
);
}
async function streamSegment(
prompt: string,
requestId: string,
onPartial: (partial: string, segmentSoFar: string) => void,
): Promise<string> {
if (!inference) {
throw new Error('MediaPipe backend is not initialized');
}
let segmentText = '';
const llm = inference;
await new Promise<void>((resolve, reject) => {
try {
llm.generateResponse(prompt, (partial: string, done: boolean) => {
if (isCancelled(requestId)) {
resolve();
return;
}
if (partial.length > 0) {
segmentText += partial;
onPartial(partial, segmentText);
}
if (done) {
resolve();
}
});
} catch (error) {
reject(error);
}
});
return segmentText;
}
function postCancelledDone(
requestId: string,
segments: number,
firstPromptTokens: number,
lastOutputTokens: number,
): void {
postMessage({
type: 'generate_done',
requestId,
stats: {
segments: Math.max(segments, 1),
promptTokens: firstPromptTokens,
outputTokens: lastOutputTokens,
continued: segments > 1,
truncated: false,
rawOutput: latestCombinedOutput,
cancelled: true,
} satisfies GenerationStats,
} satisfies LlmWorkerResponse);
clearGenerationSession();
}
async function ensureInferenceFitsPrompt(
modelFilename: string,
prompt: string,
decode: DecodeParams,
chainOfThought: boolean,
requestId: string,
progressMessage: string,
): Promise<{ effectiveDecode: DecodeParams; promptTokens: number }> {
let effectiveDecode = buildEffectiveDecode(decode, chainOfThought);
await loadInference(modelFilename, effectiveDecode, requestId, progressMessage);
const promptTokens = countTokens(prompt) ?? 0;
const requiredMaxTokens = resolvePromptAwareMaxTokens(promptTokens, decode, chainOfThought);
if (requiredMaxTokens > effectiveDecode.maxTokens) {
effectiveDecode = { ...effectiveDecode, maxTokens: requiredMaxTokens };
await loadInference(
modelFilename,
effectiveDecode,
requestId,
`Expanding token budget (${promptTokens} prompt + output room)…`,
);
}
return { effectiveDecode, promptTokens };
}
async function generateResponse(
userText: string,
promptOptions: PromptOptions,
decode: DecodeParams,
requestId: string,
rawConversationPrompt?: string,
): Promise<void> {
if (!loadedModelFilename) {
throw new Error('LLM worker is not initialized');
}
activeGenerateRequestId = requestId;
cancelRequestedFor = null;
latestCombinedOutput = '';
const chainOfThought = promptOptions.chainOfThought;
let combinedOutput = '';
let continuationAttempt = 0;
let segments = 0;
let firstPromptTokens = 0;
let lastOutputTokens = 0;
let effectiveDecode = buildEffectiveDecode(decode, chainOfThought);
while (true) {
if (isCancelled(requestId)) {
postCancelledDone(requestId, segments, firstPromptTokens, lastOutputTokens);
return;
}
const segmentNumber = segments + 1;
const prompt =
segments === 0
? rawConversationPrompt ?? formatGemmaPrompt(userText, promptOptions)
: formatGemmaContinuationPrompt(userText, combinedOutput, promptOptions);
const fit = await ensureInferenceFitsPrompt(
loadedModelFilename,
prompt,
decode,
chainOfThought,
requestId,
segments === 0 ? 'Applying decode settings…' : 'Adjusting token budget for continuation…',
);
effectiveDecode = fit.effectiveDecode;
if (!inference) {
throw new Error('MediaPipe backend is not initialized');
}
postMessage({
type: 'segment_start',
requestId,
segment: segmentNumber,
} satisfies LlmWorkerResponse);
if (segments > 0) {
postMessage({
type: 'generate_progress',
requestId,
message: `Continuing answer (part ${segments + 1})…`,
} satisfies LlmWorkerResponse);
}
const promptTokens = fit.promptTokens;
if (segments === 0) {
firstPromptTokens = promptTokens;
}
const baseBeforeSegment = combinedOutput;
let emittedLength = combinedOutput.length;
const segmentRaw = await streamSegment(prompt, requestId, (_partial, segmentSoFar) => {
const merged =
segments === 0
? segmentSoFar
: mergeContinuationOutput(baseBeforeSegment, segmentSoFar, chainOfThought);
const delta = merged.slice(emittedLength);
emittedLength = merged.length;
if (delta.length > 0) {
postMessage({
type: 'token',
requestId,
partial: delta,
done: false,
segment: segmentNumber,
} satisfies LlmWorkerResponse);
}
});
if (isCancelled(requestId)) {
combinedOutput =
segments === 0
? segmentRaw
: mergeContinuationOutput(baseBeforeSegment, segmentRaw, chainOfThought);
latestCombinedOutput = combinedOutput;
postCancelledDone(requestId, segments + 1, firstPromptTokens, lastOutputTokens);
return;
}
combinedOutput =
segments === 0
? segmentRaw
: mergeContinuationOutput(baseBeforeSegment, segmentRaw, chainOfThought);
latestCombinedOutput = combinedOutput;
segments += 1;
const totalTokens = countTokens(`${prompt}${segmentRaw}`) ?? promptTokens;
lastOutputTokens = Math.max(0, totalTokens - promptTokens);
const budgetHit = hitTokenBudgetLimit(promptTokens, totalTokens, effectiveDecode.maxTokens);
const answerText = extractAnswerText(combinedOutput, chainOfThought);
if (
!shouldContinueGeneration({
hitTokenLimit: budgetHit,
answerText,
continuationAttempt,
})
) {
const stillTruncated = looksLikeTruncatedAnswer(answerText);
postMessage({
type: 'token',
requestId,
partial: '',
done: true,
segment: segmentNumber,
} satisfies LlmWorkerResponse);
postMessage({
type: 'generate_done',
requestId,
stats: {
segments,
promptTokens: firstPromptTokens,
outputTokens: lastOutputTokens,
continued: segments > 1,
truncated: stillTruncated && budgetHit,
rawOutput: combinedOutput,
} satisfies GenerationStats,
} satisfies LlmWorkerResponse);
clearGenerationSession();
return;
}
continuationAttempt += 1;
}
}
function cancelGeneration(requestId: string): void {
if (activeGenerateRequestId !== requestId) {
return;
}
cancelRequestedFor = requestId;
inference?.cancelProcessing();
}
function postError(requestId: string, type: 'init_error' | 'generate_error', error: unknown): void {
const message = error instanceof Error ? error.message : String(error);
postMessage({ type, requestId, error: message } satisfies LlmWorkerResponse);
}
self.onmessage = async (event: MessageEvent<LlmWorkerRequest>) => {
const message = event.data;
try {
if (message.type === 'init') {
const initMs = await initInference(
message.modelFilename,
message.decode,
message.promptOptions.chainOfThought,
message.requestId,
);
postMessage({
type: 'init_done',
requestId: message.requestId,
init_ms: initMs,
} satisfies LlmWorkerResponse);
return;
}
if (message.type === 'cancel') {
cancelGeneration(message.requestId);
return;
}
if (message.type === 'generate') {
await generateResponse(
message.userText,
message.promptOptions,
message.decode,
message.requestId,
message.rawConversationPrompt,
);
}
} catch (error) {
if (message.type === 'init') {
postError(message.requestId, 'init_error', error);
} else if (message.type === 'generate') {
postError(message.requestId, 'generate_error', error);
}
}
};

View File

@@ -0,0 +1,18 @@
{
"compilerOptions": {
"target": "ES2022",
"lib": ["ES2022", "DOM", "DOM.Iterable", "WebWorker"],
"module": "ESNext",
"moduleResolution": "bundler",
"strict": true,
"skipLibCheck": true,
"noEmit": true,
"isolatedModules": true,
"useDefineForClassFields": true,
"resolveJsonModule": true,
"paths": {
"@vkist/agent-runtime": ["../../implementation/nlp/agent_runtime/src/index.ts"]
}
},
"include": ["src"]
}

View File

@@ -0,0 +1 @@
{"root":["./src/global.d.ts","./src/main.ts","./src/lib/agentchat.ts","./src/lib/capabilityprobe.ts","./src/lib/decodebenchmark.ts","./src/lib/mediapipewasmloader.ts","./src/lib/modelvalidation.ts","./src/lib/opfsmodelstore.ts","./src/lib/prompts.ts","./src/lib/toolsmoke.ts","./src/lib/types.ts","./src/lib/memory/chatsessionstore.ts","./src/lib/memory/contextassembler.ts","./src/lib/memory/deterministicembed.ts","./src/lib/memory/embedclient.ts","./src/lib/memory/episodefactory.ts","./src/lib/memory/episoderetriever.ts","./src/lib/memory/episodestore.ts","./src/lib/memory/index.ts","./src/lib/memory/memorydb.ts","./src/lib/memory/memoryorchestrator.ts","./src/lib/memory/types.ts","./src/workers/llm.worker.ts"],"version":"5.9.3"}

View File

@@ -0,0 +1,26 @@
import { defineConfig } from 'vite';
import path from 'node:path';
export default defineConfig({
server: {
port: 5174,
strictPort: true,
proxy: {
'/api': {
target: 'http://127.0.0.1:8000',
changeOrigin: true,
},
},
},
worker: {
format: 'es',
},
resolve: {
alias: {
'@vkist/agent-runtime': path.resolve(
__dirname,
'../../implementation/nlp/agent_runtime/src/index.ts',
),
},
},
});