update the codebase poc ver1

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/README.md

@@ -0,0 +1,78 @@
+# Semantic Chunking & Supabase Upload
+
+Implements Stage 3 (semantic chunking) and Stage 4 (EmbeddingGemma embed + Supabase upsert) from `knowledge/spec/ingestion/`.
+
+## Layout
+
+```
+implementation/ingestion/
+├── assemble.py           # Book-aware logical-unit assembly
+├── chunker.py            # Header pre-split + semantic breakpoint split
+├── config.py             # Paths, chunker defaults, DB mapping
+├── embedding.py          # EmbeddingGemma ONNX wrapper (use_gemma.py aligned)
+├── metadata_parser.py    # source_path filename parsing
+├── pipeline.py           # CLI orchestrator
+├── sqlite_store.py       # Local semantic_chunks staging in corpus_db
+├── supabase_uploader.py  # Upsert to knowledge.* Supabase schema
+└── requirements.txt
+```
+
+## Prerequisites
+
+1. Checkpoint databases in `knowledge/corpus_db/v1/` with `processed_chunks` populated.
+2. Supabase migrations applied (`implementation/supabase/migrations/`).
+3. Secrets at `PILOT_PROJECT/secrets/aws_secret/.env`:
+
+```bash
+SUPABASE_URL=https://<project-ref>.supabase.co
+SUPABASE_PUBLISHABLE_KEY=<publishable-key>
+```
+
+4. **Upload credentials** (choose one):
+
+| Method | Required setup |
+|--------|----------------|
+| PostgreSQL (recommended) | `SUPABASE_DB_URL` or `DATABASE_URL` from Supabase Dashboard → Database → Connection string |
+| Supabase REST | Expose `knowledge` schema in Dashboard → Project Settings → API → Exposed schemas, plus `SUPABASE_SERVICE_ROLE_KEY` |
+| Supabase CLI | Run `supabase link` under `implementation/supabase/` (uses `supabase db query --linked`) |
+
+`SUPABASE_PUBLISHABLE_KEY` alone is not sufficient for ingestion writes.
+
+4. Conda environment:
+
+```bash
+conda activate vkist_ultra
+pip install -r implementation/ingestion/requirements.txt
+```
+
+## Usage
+
+From `knowledge/implementation/`:
+
+```bash
+# Full pipeline for one book (chunk + embed + upload)
+python -m ingestion.pipeline --books mor
+
+# All books
+python -m ingestion.pipeline --books mor oho tny ado
+
+# Chunk only (writes semantic_chunks to local SQLite)
+python -m ingestion.pipeline --books mor --chunk-only
+
+# Upload pending chunks only
+python -m ingestion.pipeline --books mor --upload-only
+
+# Re-chunk existing logical units
+python -m ingestion.pipeline --books mor --force
+```
+
+## Chunker version
+
+Active version: `header+semantic_v1` (see `semantic_chunking_spec.md`).
+
+## References
+
+- `spec/ingestion/semantic_chunking_spec.md`
+- `spec/ingestion/corpus_profiles.md`
+- `spec/ingestion/schema.md`
+- `tests/use_gemma.py` — embedding instruction prefixes

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Knowledge ingestion pipeline: semantic chunking and Supabase upload."""

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/__main__.py

@@ -0,0 +1,6 @@
+"""Module entrypoint for `python -m ingestion.pipeline`."""
+
+from ingestion.pipeline import main
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/assemble.py

@@ -0,0 +1,373 @@
+"""Book-aware logical-unit assembly from processed_chunks rows."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .config import BATCH_BOUNDARY_MARKER, GLUED_WORD_RE, MOR_TOC_PATH, IMAGE_MARKER_RE, PROTECTED_BLOCK_RE, TABLE_BLOCK_RE, STRUCTURAL_LINE_RE, HYPHENATION_BREAK_RE, SENTENCE_END_RE
+from .metadata_parser import SourceMetadata, infer_book_id, parse_source_path
+
+@dataclass
+class ProcessedChunkRow:
+    """One extraction checkpoint row."""
+
+    row_id: int
+    source_path: str
+    markdown_content: str
+    metadata: SourceMetadata
+
+
+@dataclass
+class LogicalUnit:
+    """An assembled text unit ready for header/semantic chunking."""
+
+    logical_unit_id: str
+    book_id: str
+    content: str
+    source_extraction_ids: list[int]
+    page_start: int
+    page_end: int
+    section_id: int | None = None
+    subsection_id: str | None = None
+    parent_title: str | None = None
+    assembly_metadata: dict = field(default_factory=dict)
+
+
+def load_mor_titles(toc_path: Path = MOR_TOC_PATH) -> dict[str, str]:
+    """Build subsection_id -> title lookup from MOR TOC JSON."""
+    with toc_path.open(encoding="utf-8") as handle:
+        payload = json.load(handle)
+
+    titles: dict[str, str] = {}
+    for section in payload.get("sections", []):
+        for subsection in section.get("subsections", []):
+            title = subsection.get("title", "")
+            match = re.match(r"^(\d+\.\d+)\b", title)
+            if match:
+                titles[match.group(1)] = title
+    return titles
+
+
+def _sort_key(row: ProcessedChunkRow) -> tuple:
+    meta = row.metadata
+    return (
+        meta.page_start,
+        meta.batch_index if meta.batch_index is not None else 0,
+        row.row_id,
+    )
+
+
+def _slugify_header(header: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", header.lower()).strip("_")
+    return slug[:80] or "untitled"
+
+
+def _logical_unit_anchor(start: int, header_title: str) -> str:
+    """Return a stable short suffix for repeated header titles in loose-tier books."""
+    digest = hashlib.sha256(f"{start}:{header_title}".encode("utf-8")).hexdigest()
+    return digest[:8]
+
+def _wrap_markdown_tables(text: str) -> str:
+    def replacer(match: re.Match[str]) -> str:
+        table = match.group(0).strip()
+        # Skip if already wrapped
+        if table.startswith("<table>"):
+            return table
+        return f"<table>\n{table}\n</table>"
+    return TABLE_BLOCK_RE.sub(replacer, text)
+
+def _is_prose_continuation(
+    previous_line: str,
+    next_line: str,
+    *,
+    across_blank: bool = False,
+) -> bool:
+    """Return True when next_line likely continues previous_line across a break."""
+    if STRUCTURAL_LINE_RE.match(next_line):
+        return False
+    if next_line[0].islower():
+        return True
+    if across_blank:
+        return False
+    return not SENTENCE_END_RE.search(previous_line.rstrip())
+
+def _normalize_newlines(text: str) -> str:
+    """Fix Docling/PDF soft wraps while preserving markdown structure."""
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    text = HYPHENATION_BREAK_RE.sub(r"\1\2", text)
+
+    lines = text.split("\n")
+    out: list[str] = []
+    index = 0
+    while index < len(lines):
+        line = lines[index].rstrip()
+        if not line:
+            out.append("")
+            index += 1
+            continue
+
+        if STRUCTURAL_LINE_RE.match(line):
+            out.append(line)
+            index += 1
+            continue
+
+        while index + 1 < len(lines):
+            blank_skip = 0
+            next_index = index + 1
+            if not lines[next_index].strip():
+                blank_skip = 1
+                next_index += 1
+            if next_index >= len(lines):
+                break
+
+            nxt = lines[next_index].strip()
+            if not nxt:
+                break
+            if blank_skip and not _is_prose_continuation(line, nxt, across_blank=True):
+                break
+            if STRUCTURAL_LINE_RE.match(nxt):
+                break
+            if not blank_skip and not _is_prose_continuation(line, nxt):
+                break
+
+            line = f"{line} {nxt}"
+            index = next_index
+            lines[index] = ""
+
+        out.append(re.sub(r" {2,}", " ", line))
+        index += 1
+
+    normalized = "\n".join(out)
+    normalized = re.sub(r"\n{3,}", "\n\n", normalized)
+    return normalized.strip()
+
+def _split_loose_book_stream(
+    rows: list[ProcessedChunkRow],
+) -> list[LogicalUnit]:
+    """Split a stitched loose-tier book stream into header-based logical units."""
+    if not rows:
+        return []
+
+    book_id = rows[0].metadata.book_id
+    stitched_parts: list[str] = []
+    extraction_ids: list[int] = []
+    for row in rows:
+        stitched_parts.append(row.markdown_content.strip())
+        extraction_ids.append(row.row_id)
+
+    book_stream = _clean_assembled_text(
+        BATCH_BOUNDARY_MARKER.join(stitched_parts).replace(BATCH_BOUNDARY_MARKER, "\n\n"),
+        book_id=book_id,
+    )
+    page_start = min(row.metadata.page_start for row in rows)
+    page_end = max(row.metadata.page_end for row in rows)
+
+    header_pattern = re.compile(r"^(#{1,2})\s+(.+)$", re.MULTILINE)
+    matches = list(header_pattern.finditer(book_stream))
+    if not matches:
+        return [
+            LogicalUnit(
+                logical_unit_id=f"{book_id}_book",
+                book_id=book_id,
+                content=book_stream.strip(),
+                source_extraction_ids=extraction_ids,
+                page_start=page_start,
+                page_end=page_end,
+                assembly_metadata={"strategy": "full_book_stream"},
+            )
+        ]
+
+    units: list[LogicalUnit] = []
+    for index, match in enumerate(matches):
+        start = match.start()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(book_stream)
+        section_text = book_stream[start:end].strip()
+        if not section_text:
+            continue
+
+        header_title = match.group(2).strip()
+        header_level = len(match.group(1))
+        unit_suffix = _slugify_header(header_title)
+        anchor = _logical_unit_anchor(start, header_title)
+        logical_unit_id = f"{book_id}_h{header_level}_{unit_suffix}_{anchor}"
+
+        units.append(
+            LogicalUnit(
+                logical_unit_id=logical_unit_id,
+                book_id=book_id,
+                content=section_text,
+                source_extraction_ids=extraction_ids,
+                page_start=page_start,
+                page_end=page_end,
+                parent_title=header_title,
+                assembly_metadata={"strategy": "header_boundary", "header_level": header_level},
+            )
+        )
+    return units
+
+def assemble_logical_units(
+    rows: list[tuple[int, str, str]],
+    book_id: str,
+    mor_titles: dict[str, str] | None = None,
+) -> list[LogicalUnit]:
+    """Assemble checkpoint rows into logical units according to corpus_profiles.md."""
+    parsed_rows = [
+        ProcessedChunkRow(
+            row_id=row_id,
+            source_path=source_path,
+            markdown_content=markdown_content,
+            metadata=parse_source_path(source_path, book_id=book_id),
+        )
+        for row_id, source_path, markdown_content in rows
+    ]
+    parsed_rows.sort(key=_sort_key)
+
+    if book_id in {"ado", "tny"}:
+        return _split_loose_book_stream(parsed_rows)
+
+    if book_id == "oho":
+        grouped: dict[int, list[ProcessedChunkRow]] = {}
+        for row in parsed_rows:
+            section_id = row.metadata.section_id
+            assert section_id is not None
+            grouped.setdefault(section_id, []).append(row)
+
+        units: list[LogicalUnit] = []
+        for section_id in sorted(grouped):
+            section_rows = grouped[section_id]
+            stitched = BATCH_BOUNDARY_MARKER.join(
+                row.markdown_content.strip() for row in section_rows
+            )
+            content = _clean_assembled_text(
+                stitched.replace(BATCH_BOUNDARY_MARKER, "\n\n"),
+                book_id=book_id,
+            )
+            units.append(
+                LogicalUnit(
+                    logical_unit_id=f"sec_{section_id}",
+                    book_id=book_id,
+                    content=content.strip(),
+                    source_extraction_ids=[row.row_id for row in section_rows],
+                    page_start=min(row.metadata.page_start for row in section_rows),
+                    page_end=max(row.metadata.page_end for row in section_rows),
+                    section_id=section_id,
+                    assembly_metadata={"strategy": "section_stitch", "batch_count": len(section_rows)},
+                )
+            )
+        return units
+
+    if book_id == "mor":
+        titles = mor_titles or load_mor_titles()
+        units = []
+        for row in parsed_rows:
+            subsection_id = row.metadata.subsection_id
+            assert subsection_id is not None
+            units.append(
+                LogicalUnit(
+                    logical_unit_id=f"sub_{subsection_id}",
+                    book_id=book_id,
+                    content=_clean_assembled_text(row.markdown_content, book_id=book_id),
+                    source_extraction_ids=[row.row_id],
+                    page_start=row.metadata.page_start,
+                    page_end=row.metadata.page_end,
+                    section_id=row.metadata.section_id,
+                    subsection_id=subsection_id,
+                    parent_title=titles.get(subsection_id),
+                    assembly_metadata={"strategy": "one_row_one_unit"},
+                )
+            )
+        return units
+
+    raise ValueError(f"Unsupported book_id for assembly: {book_id}")
+
+def _normalize_table_token_boundaries(text: str) -> str:
+      """Ensure block-level separation around wrapped tables."""
+      # </table> glued to following content (e.g. </table>## Juvenile SLE)
+      text = re.sub(r"</table>(?=\S)", "</table>\n\n", text)
+      # prose glued to opening tag (e.g. sentence<table>)
+      text = re.sub(r"(?<=\S)<table>", "\n\n<table>", text)
+      # optional: header glued before table (e.g. ## Title<table>)
+      text = re.sub(r"(#{1,6}\s[^\n]+)<table>", r"\1\n\n<table>", text)
+      return text
+
+def _normalize_header_lines(text: str) -> str:
+      """Collapse internal spacing in markdown headers."""
+      def _collapse_header(match: re.Match[str]) -> str:
+          prefix, title = match.group(1), match.group(2)
+          return f"{prefix}{' '.join(title.split())}"
+      return re.sub(
+          r"^(#{1,6}\s+)(.+)$",
+          _collapse_header,
+          text,
+          flags=re.MULTILINE,
+      )
+
+def _collapse_inline_whitespace(line: str) -> str:
+      # Handles spaces, tabs, nbsp, etc.
+      return " ".join(line.split())
+
+def _normalize_middle_dot_bullets(text: str) -> str:
+      # "plaques. · Lesions" → "plaques.\n\n- Lesions"  (optional list form)
+      text = re.sub(r"\s*·\s*", "\n- ", text)
+      # OR lighter touch — just normalize spacing:
+      # text = re.sub(r"\s*·\s*", " · ", text)
+      return text
+
+def _repair_glued_words(line: str) -> str:
+      def fix(m: re.Match[str]) -> str:
+          prefix, suffix = m.group(1), m.group(2)
+          # Avoid breaking real words: "mayo", "island", "androgen"
+          if suffix.lower() in {"o", "be", "occur", "also", "not"}:  # extend as needed
+              return f"{prefix} {suffix}"
+          return m.group(0)
+      return GLUED_WORD_RE.sub(fix, line)
+
+def _normalize_prose_spacing(text: str) -> str:
+      placeholders: dict[str, str] = {}
+      def _mask(match: re.Match[str]) -> str:
+          key = f"__PROTECTED_{len(placeholders)}__"
+          placeholders[key] = match.group(0)
+          return key
+      masked = PROTECTED_BLOCK_RE.sub(_mask, text)
+      lines = []
+      for line in masked.split("\n"):
+          if not line.strip():
+              lines.append("")
+              continue
+          if STRUCTURAL_LINE_RE.match(line):
+              lines.append(line)
+              continue
+          line = _collapse_inline_whitespace(line)
+          line = _repair_glued_words(line)  # optional
+          lines.append(line)
+      restored = "\n".join(lines)
+      for key, value in placeholders.items():
+          restored = restored.replace(key, value)
+      return _normalize_middle_dot_bullets(restored)  # optional
+
+def _clean_assembled_text(text: str, book_id: str | None = None) -> str:
+      cleaned = IMAGE_MARKER_RE.sub("", text)
+      cleaned = _normalize_newlines(cleaned)
+      cleaned = _wrap_markdown_tables(cleaned)
+      cleaned = _normalize_table_token_boundaries(cleaned)   # </table>## fix
+      cleaned = _normalize_prose_spacing(cleaned)            # NEW — intra-word spaces
+      cleaned = _normalize_header_lines(cleaned)
+      cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+      return cleaned.strip()
+
+
+def rows_for_book(
+    all_rows: list[tuple[int, str, str]],
+    book_id: str,
+) -> list[tuple[int, str, str]]:
+    """Filter mixed checkpoint rows to a single book."""
+    filtered = []
+    for row in all_rows:
+        inferred = infer_book_id(row[1])
+        if inferred == book_id:
+            filtered.append(row)
+    return filtered

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/chunker.py

@@ -0,0 +1,399 @@
+"""Header and semantic chunking for retrieval-ready markdown units."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import uuid
+from dataclasses import dataclass
+from typing import Literal
+
+import numpy as np
+from langchain_text_splitters import MarkdownHeaderTextSplitter
+
+from .assemble import LogicalUnit, _wrap_markdown_tables
+from .config import (
+    CAPTION_LINE_RE,
+    CHUNKER_VERSION,
+    ChunkerConfig,
+    CODE_FENCE_RE,
+    DEFAULT_CHUNKER_CONFIG,
+    SENTENCE_SPLIT_RE,
+    TABLE_BLOCK_RE,
+    WRAPPED_TABLE_RE,
+)
+from .embedding import GemmaEmbedder, cosine_similarity
+
+logger = logging.getLogger(__name__)
+
+BlockType = Literal["prose", "table"]
+
+
+@dataclass(frozen=True)
+class ContentBlock:
+    """A prose or table segment extracted before chunking."""
+
+    block_type: BlockType
+    text: str
+    context: str | None = None
+
+
+@dataclass(frozen=True)
+class SemanticChunkRecord:
+    """One chunk ready for SQLite staging and Supabase upsert."""
+
+    chunk_uuid: str
+    logical_unit_id: str
+    chunk_index: int
+    content: str
+    token_count: int
+    book_id: str
+    section_id: int | None
+    subsection_id: str | None
+    parent_title: str | None
+    page_start: int | None
+    page_end: int | None
+    source_extraction_ids: list[int]
+    chunker_version: str
+    content_hash: str
+    embed_content: str | None = None
+
+
+def content_hash(content: str) -> str:
+    """Return SHA-256 hex digest for deduplication."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def embedding_text(record: SemanticChunkRecord) -> str:
+    """Return chunk body for retrieval embedding (title is passed separately to the embedder)."""
+    if record.embed_content:
+        return record.embed_content
+    if "<table>" in record.content:
+        return _table_embed_content(record.content, context=None)
+    return record.content
+
+
+def _headers_for_book(book_id: str) -> list[tuple[str, str]]:
+    if book_id in {"ado", "tny"}:
+        return [("#", "h1"), ("##", "h2"), ("###", "h3")]
+    return [("##", "h2"), ("###", "h3")]
+
+
+def _peel_trailing_caption(prose: str) -> tuple[str, str | None]:
+    """Move a trailing table/figure caption from prose onto the next table block."""
+    lines = prose.rstrip().split("\n")
+    if not lines:
+        return prose, None
+
+    last = lines[-1].strip()
+    if CAPTION_LINE_RE.match(last):
+        remaining = "\n".join(lines[:-1]).strip()
+        return remaining, last
+    return prose, None
+
+
+def _table_context_from_prose(prose: str) -> str | None:
+    """Return caption or last sentence from prose immediately before a table."""
+    prose = prose.strip()
+    if not prose:
+        return None
+
+    _, caption = _peel_trailing_caption(prose)
+    if caption:
+        return caption
+
+    sentences = _split_sentences(prose)
+    last_sentence = sentences[-1].strip() if sentences else ""
+    return last_sentence or None
+
+
+def _needs_table_wrap(text: str) -> bool:
+    """Return True when pipe markdown tables exist without assembly wrappers."""
+    if WRAPPED_TABLE_RE.search(text):
+        return False
+    return TABLE_BLOCK_RE.search(text) is not None
+
+
+def _prepare_text_for_extract(text: str) -> str:
+    """Normalize table markers only when assembly has not already wrapped them."""
+    stripped = text.strip()
+    if not stripped:
+        return ""
+    if _needs_table_wrap(stripped):
+        return _wrap_markdown_tables(stripped)
+    return stripped
+
+
+def extract_blocks(text: str) -> list[ContentBlock]:
+    """Split assembled markdown into alternating prose and table blocks."""
+    normalized = _prepare_text_for_extract(text)
+    if not normalized:
+        return []
+
+    if not WRAPPED_TABLE_RE.search(normalized):
+        return [ContentBlock(block_type="prose", text=normalized)]
+
+    blocks: list[ContentBlock] = []
+    position = 0
+
+    for match in WRAPPED_TABLE_RE.finditer(normalized):
+        prose_before = normalized[position : match.start()].strip()
+        caption: str | None = None
+        if prose_before:
+            prose_before, caption = _peel_trailing_caption(prose_before)
+            if prose_before:
+                blocks.append(ContentBlock(block_type="prose", text=prose_before))
+
+        table_text = match.group(0).strip()
+        if caption:
+            table_text = f"{caption}\n\n{table_text}"
+
+        context = caption or _table_context_from_prose(prose_before)
+        blocks.append(
+            ContentBlock(
+                block_type="table",
+                text=table_text,
+                context=context,
+            )
+        )
+        position = match.end()
+
+    trailing = normalized[position:].strip()
+    if trailing:
+        blocks.append(ContentBlock(block_type="prose", text=trailing))
+
+    return blocks
+
+
+def _split_sentences(text: str) -> list[str]:
+    protected = text
+    placeholders: dict[str, str] = {}
+
+    for index, match in enumerate(CODE_FENCE_RE.finditer(text)):
+        key = f"__CODE_BLOCK_{index}__"
+        placeholders[key] = match.group(0)
+        protected = protected.replace(match.group(0), key, 1)
+
+    for index, match in enumerate(TABLE_BLOCK_RE.finditer(protected)):
+        key = f"__TABLE_BLOCK_{index}__"
+        placeholders[key] = match.group(0)
+        protected = protected.replace(match.group(0), key, 1)
+
+    parts = SENTENCE_SPLIT_RE.split(protected.strip())
+    restored = []
+    for part in parts:
+        chunk = part.strip()
+        if not chunk:
+            continue
+        for key, value in placeholders.items():
+            chunk = chunk.replace(key, value)
+        restored.append(chunk)
+    return restored or [text.strip()]
+
+
+def _header_split_text(text: str, book_id: str) -> list[tuple[str, dict]]:
+    splitter = MarkdownHeaderTextSplitter(
+        headers_to_split_on=_headers_for_book(book_id),
+        strip_headers=False,
+    )
+    docs = splitter.split_text(text)
+    sections: list[tuple[str, dict]] = []
+    for doc in docs:
+        section_text = doc.page_content.strip()
+        if not section_text:
+            continue
+        sections.append((section_text, dict(doc.metadata)))
+    if not sections:
+        sections.append((text.strip(), {}))
+    return sections
+
+
+def _resolve_parent_title(unit: LogicalUnit, header_meta: dict) -> str | None:
+    if unit.parent_title:
+        return unit.parent_title
+    for key in ("h3", "h2", "h1"):
+        if header_meta.get(key):
+            return header_meta[key]
+    return None
+
+
+def _semantic_split_text(
+    text: str,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[str]:
+    sentences = _split_sentences(text)
+    if len(sentences) <= 1:
+        return [text.strip()]
+
+    embeddings = [embedder.embed_clustering(sentence) for sentence in sentences]
+    distances = [
+        1.0 - cosine_similarity(embeddings[index], embeddings[index + 1])
+        for index in range(len(embeddings) - 1)
+    ]
+
+    if not distances:
+        return [text.strip()]
+
+    threshold = float(np.percentile(distances, config.breakpoint_threshold_amount))
+    breakpoints = [0]
+    for index, distance in enumerate(distances):
+        if distance >= threshold:
+            breakpoints.append(index + 1)
+    breakpoints.append(len(sentences))
+
+    chunks: list[str] = []
+    for start, end in zip(breakpoints[:-1], breakpoints[1:]):
+        chunk_text = " ".join(sentences[start:end]).strip()
+        if chunk_text:
+            chunks.append(chunk_text)
+    return chunks or [text.strip()]
+
+
+def _split_oversized_piece(
+    text: str,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[str]:
+    token_count = embedder.count_tokens(text)
+    hard_cap = int(config.max_tokens * config.hard_cap_multiplier)
+    if token_count <= config.max_tokens:
+        return [text]
+
+    if token_count <= hard_cap and TABLE_BLOCK_RE.search(text):
+        return [text]
+
+    return _semantic_split_text(text, embedder, config)
+
+
+def _chunk_prose_block(
+    prose_text: str,
+    unit: LogicalUnit,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[tuple[str, str | None, BlockType, str | None]]:
+    """Header- and semantic-split one prose block; never merges with tables."""
+    pieces: list[tuple[str, str | None, BlockType, str | None]] = []
+    header_sections = _header_split_text(prose_text, unit.book_id)
+
+    for section_text, header_meta in header_sections:
+        parent_title = _resolve_parent_title(unit, header_meta)
+        if unit.book_id == "mor" and embedder.count_tokens(section_text) <= config.max_tokens:
+            pieces.append((section_text, parent_title, "prose", None))
+            continue
+
+        for piece in _split_oversized_piece(section_text, embedder, config):
+            pieces.append((piece, parent_title, "prose", None))
+
+    return pieces
+
+
+def _table_embed_content(
+    table_content: str,
+    *,
+    context: str | None,
+) -> str:
+    """Build table chunk body for retrieval embedding (title uses the embedder prefix)."""
+    if context and context not in table_content:
+        return f"{context.strip()}\n\n{table_content.strip()}"
+    return table_content.strip()
+
+
+def _merge_tiny_prose_pieces(
+    pieces: list[tuple[str, str | None, BlockType, str | None]],
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[tuple[str, str | None, BlockType, str | None]]:
+    """Merge undersized prose chunks only; tables stay isolated."""
+    if not pieces:
+        return []
+
+    merged: list[tuple[str, str | None, BlockType, str | None]] = []
+    carry_text: str | None = None
+    carry_title: str | None = None
+
+    for text, title, block_type, context in pieces:
+        if block_type == "table":
+            if carry_text is not None:
+                merged.append((carry_text, carry_title, "prose", None))
+                carry_text, carry_title = None, None
+            merged.append((text, title, "table", context))
+            continue
+
+        if carry_text is None:
+            carry_text, carry_title = text, title
+            continue
+
+        carry_tokens = embedder.count_tokens(carry_text)
+        piece_tokens = embedder.count_tokens(text)
+        max_merge_size = int(config.max_tokens * 1.25)
+        if carry_tokens < config.min_tokens and carry_tokens + piece_tokens <= max_merge_size:
+            carry_text = f"{carry_text}\n\n{text}".strip()
+            if not carry_title:
+                carry_title = title
+        else:
+            merged.append((carry_text, carry_title, "prose", None))
+            carry_text, carry_title = text, title
+
+    if carry_text is not None:
+        merged.append((carry_text, carry_title, "prose", None))
+
+    return merged
+
+
+def chunk_logical_unit(
+    unit: LogicalUnit,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig = DEFAULT_CHUNKER_CONFIG,
+    chunker_version: str = CHUNKER_VERSION,
+) -> list[SemanticChunkRecord]:
+    """Chunk one logical unit using prose/table split, header pre-split, and semantic refinement."""
+    content_blocks = extract_blocks(unit.content)
+    raw_pieces: list[tuple[str, str | None, BlockType, str | None]] = []
+
+    for block in content_blocks:
+        if block.block_type == "table":
+            raw_pieces.append((block.text, unit.parent_title, "table", block.context))
+            continue
+        raw_pieces.extend(_chunk_prose_block(block.text, unit, embedder, config))
+
+    merged_pieces = _merge_tiny_prose_pieces(raw_pieces, embedder, config)
+
+    records: list[SemanticChunkRecord] = []
+    seen_hashes: set[str] = set()
+
+    for index, (text, parent_title, block_type, context) in enumerate(merged_pieces):
+        normalized = text.strip()
+        if not normalized:
+            continue
+        digest = content_hash(normalized)
+        if digest in seen_hashes:
+            logger.debug("Skipping duplicate chunk content in %s", unit.logical_unit_id)
+            continue
+        seen_hashes.add(digest)
+
+        resolved_title = parent_title or unit.parent_title
+        embed_content = None
+        if block_type == "table":
+            embed_content = _table_embed_content(normalized, context=context)
+
+        records.append(
+            SemanticChunkRecord(
+                chunk_uuid=str(uuid.uuid4()),
+                logical_unit_id=unit.logical_unit_id,
+                chunk_index=index,
+                content=normalized,
+                token_count=embedder.count_tokens(normalized),
+                book_id=unit.book_id,
+                section_id=unit.section_id,
+                subsection_id=unit.subsection_id,
+                parent_title=resolved_title,
+                page_start=unit.page_start,
+                page_end=unit.page_end,
+                source_extraction_ids=list(unit.source_extraction_ids),
+                chunker_version=chunker_version,
+                content_hash=digest,
+                embed_content=embed_content,
+            )
+        )
+    return records

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/config.py

@@ -0,0 +1,100 @@
+"""Configuration for the knowledge ingestion pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import re
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parents[2]
+IMPLEMENTATION_ROOT = Path(__file__).resolve().parents[1]
+CORPUS_DB_DIR = KNOWLEDGE_ROOT / "corpus_db" / "v1"
+CORPUS_ROOT = KNOWLEDGE_ROOT / "corpus"
+MODEL_DIR = IMPLEMENTATION_ROOT / "model" / "gemma_final"
+MOR_TOC_PATH = CORPUS_ROOT / "pdf" / "mor" / "toc_structure.json"
+IMAGE_MARKER_RE = re.compile(r"^\s*<!--\s*image\s*-->\s*$", re.MULTILINE | re.IGNORECASE)
+DEFAULT_ENV_PATH = (
+    Path(__file__).resolve().parents[6]
+    / "secrets"
+    / "aws_secret"
+    / ".env"
+)
+TABLE_BLOCK_RE = re.compile(
+    r"(?:^\|.+\|\s*\n(?:^\|[-:\s|]+\|\s*\n)?(?:^\|.+\|\s*\n?)+)",
+    re.MULTILINE,
+)
+
+WRAPPED_TABLE_RE = re.compile(r"<table>[\s\S]*?</table>", re.MULTILINE)
+
+CAPTION_LINE_RE = re.compile(
+    r"^(?:Table|Figure|Fig\.)\s+[\dIVXivx]+",
+    re.IGNORECASE,
+)
+
+CODE_FENCE_RE = re.compile(r"```[\s\S]*?```", re.MULTILINE)
+
+SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"(])")
+
+CHUNKER_VERSION = "header+semantic_v2"
+EMBEDDING_MODEL_NAME = "EmbeddingGemma"
+EMBEDDING_MODEL_VERSION = "1"
+EMBEDDING_DIMENSIONS = 768
+EDITION_LABEL = "v1"
+
+BATCH_BOUNDARY_MARKER = "\n\n<!-- batch-boundary -->\n\n"
+
+BOOK_DB_FILES: dict[str, str] = {
+    "ado": "ado_ingestion_corpus.db",
+    "tny": "tny_ingestion_corpus.db",
+    "oho": "oho_ingestion_corpus.db",
+    "mor": "mor_ingestion_corpus.db",
+}
+
+ADO_LEGACY_FILENAME_RE = re.compile(
+      r"^chunk_(?P<batch_index>\d+)-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+  )
+
+PROTECTED_BLOCK_RE = re.compile(
+    r"(<table>[\s\S]*?</table>|```[\s\S]*?```|"
+    r"(?:^\|.+\|\s*\n(?:^\|[-:\s|]+\|\s*\n)?(?:^\|.+\|\s*\n?)+))",
+    re.MULTILINE,
+)
+
+GLUED_WORD_RE = re.compile(
+      r"\b(may|can|are|is|and|the)([a-z]{3,})\b",
+      re.IGNORECASE,
+  )
+
+CONSTRAINT_TIERS: dict[str, str] = {
+    "ado": "loose",
+    "tny": "loose",
+    "oho": "section",
+    "mor": "subsection",
+}
+
+STRUCTURAL_LINE_RE = re.compile(r"^(#{1,6}\s|[-*+]\s|\d+\.\s|```|\|)")
+HYPHENATION_BREAK_RE = re.compile(r"(\w)-\n(\w)")
+SENTENCE_END_RE = re.compile(r'[.!?:]["\']?\s*$')
+
+
+@dataclass(frozen=True)
+class ChunkerConfig:
+    """Semantic chunking parameters from semantic_chunking_spec.md."""
+
+    max_tokens: int = 800
+    min_tokens: int = 120
+    overlap_tokens: int = 64
+    breakpoint_threshold_type: str = "percentile"
+    breakpoint_threshold_amount: float = 95.0
+    hard_cap_multiplier: float = 1.5
+
+
+DEFAULT_CHUNKER_CONFIG = ChunkerConfig()
+
+
+def resolve_corpus_db_path(book_id: str) -> Path:
+    """Return the SQLite checkpoint database path for a textbook."""
+    filename = BOOK_DB_FILES.get(book_id)
+    if filename is None:
+        raise ValueError(f"Unknown book_id: {book_id}")
+    return CORPUS_DB_DIR / filename

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/embedding.py

@@ -0,0 +1,149 @@
+"""EmbeddingGemma ONNX embedding wrapper aligned with use_gemma.py."""
+
+from __future__ import annotations
+
+import logging
+from enum import Enum
+from functools import lru_cache
+from pathlib import Path
+
+import numpy as np
+import torch
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+from transformers import AutoTokenizer
+
+from .config import EMBEDDING_DIMENSIONS, MODEL_DIR
+
+logger = logging.getLogger(__name__)
+
+
+class EmbedTask(str, Enum):
+    """EmbeddingGemma instruction-routing tasks used by this pipeline."""
+
+    CLUSTERING = "Clustering"
+    RETRIEVAL_DOCUMENT = "Retrieval-document"
+    RETRIEVAL_QUERY = "Retrieval-query"
+
+
+# Full task map from EmbeddingGemma / MTEB; pipeline uses EmbedTask subset only.
+TASK_PREFIXES: dict[str, str] = {
+    "query": "task: search result | query: ",
+    "document": "title: none | text: ",
+    "BitextMining": "task: search result | query: ",
+    "Clustering": "task: clustering | query: ",
+    "Classification": "task: classification | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-query": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: ",
+}
+
+# Backward-compatible aliases.
+AVAILABLE_TASK = TASK_PREFIXES
+DOC_PREFIX = TASK_PREFIXES["Retrieval-document"]
+QUERY_PREFIX = TASK_PREFIXES["Retrieval-query"]
+
+
+def format_embed_input(
+    text: str,
+    task: EmbedTask,
+    *,
+    title: str | None = None,
+) -> str:
+    """Apply the EmbeddingGemma instruction prefix for a pipeline role."""
+    if task == EmbedTask.RETRIEVAL_DOCUMENT:
+        title_value = title.strip() if title and title.strip() else "none"
+        return f"title: {title_value} | text: {text}"
+    return TASK_PREFIXES[task.value] + text
+
+
+class GemmaEmbedder:
+    """Local EmbeddingGemma feature extractor with task-routed instruction prefixes."""
+
+    def __init__(
+        self,
+        model_dir: Path = MODEL_DIR,
+        target_dim: int = EMBEDDING_DIMENSIONS,
+        provider: str = "CPUExecutionProvider",
+    ) -> None:
+        """Load tokenizer and ONNX model from the local model directory."""
+        if not hasattr(torch, "int4"):
+            torch.int4 = torch.int8
+
+        self.target_dim = target_dim
+        self.tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)
+        self.model = ORTModelForFeatureExtraction.from_pretrained(
+            model_dir,
+            file_name="model_fp16.onnx",
+            local_files_only=True,
+            provider=provider,
+            use_io_binding=False,
+        )
+        logger.info("Loaded EmbeddingGemma from %s", model_dir)
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens using the embedding model tokenizer."""
+        return len(
+            self.tokenizer.encode(
+                text,
+                add_special_tokens=True,
+                truncation=False,
+            )
+        )
+
+    def embed(
+        self,
+        text: str,
+        task: EmbedTask,
+        *,
+        title: str | None = None,
+    ) -> np.ndarray:
+        """Embed text with the instruction prefix for the given pipeline task."""
+        return self._embed(format_embed_input(text, task, title=title))
+
+    def embed_document(self, text: str, *, title: str | None = None) -> np.ndarray:
+        """Embed a retrieval corpus chunk (asymmetric document side)."""
+        return self.embed(text, EmbedTask.RETRIEVAL_DOCUMENT, title=title)
+
+    def embed_query(self, text: str) -> np.ndarray:
+        """Embed a retrieval search query (asymmetric query side)."""
+        return self.embed(text, EmbedTask.RETRIEVAL_QUERY)
+
+    def embed_clustering(self, text: str) -> np.ndarray:
+        """Embed text for semantic breakpoint / clustering during chunking."""
+        return self.embed(text, EmbedTask.CLUSTERING)
+
+    def _embed(self, text: str) -> np.ndarray:
+        inputs = self.tokenizer(
+            text,
+            max_length=512,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+        )
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        raw_states = outputs.last_hidden_state.numpy()
+        mean_pooled = np.mean(raw_states, axis=1)[0]
+        vector = mean_pooled[: self.target_dim]
+        norm = np.linalg.norm(vector)
+        if norm == 0:
+            return vector
+        return vector / norm
+
+
+@lru_cache(maxsize=1)
+def get_embedder() -> GemmaEmbedder:
+    """Return a process-wide singleton embedder instance."""
+    return GemmaEmbedder()
+
+
+def cosine_similarity(vector_a: np.ndarray, vector_b: np.ndarray) -> float:
+    """Compute cosine similarity between two vectors."""
+    denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)
+    if denom == 0:
+        return 0.0
+    return float(np.dot(vector_a, vector_b) / denom)

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/metadata_parser.py

@@ -0,0 +1,122 @@
+"""Parse source_path filenames into structured ingestion metadata."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import ADO_LEGACY_FILENAME_RE
+
+
+BATCH_FILENAME_RE = re.compile(
+    r"^batch-(?P<batch_index>\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+)
+OHO_FILENAME_RE = re.compile(
+    r"^sec_(?P<section_id>\d+)_batch-(?P<batch_index>\d+)_"
+    r"chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+)
+MOR_FILENAME_RE = re.compile(
+    r"^sub_(?P<subsection_id>\d+\.\d+)_chunk-(?P<page_start>\d+)-"
+    r"(?P<page_end>\d+)\.pdf$"
+)
+
+
+@dataclass(frozen=True)
+class SourceMetadata:
+    """Structured metadata extracted from a processed_chunks source_path."""
+
+    book_id: str
+    page_start: int
+    page_end: int
+    section_id: int | None
+    subsection_id: str | None
+    batch_index: int | None
+    source_filename: str
+
+
+def infer_book_id(source_path: str) -> str:
+    """Infer book_id from the source_path string."""
+    normalized = source_path.replace("\\", "/").lower()
+    for book_id in ("mor", "oho", "tny", "ado"):
+        if f"/{book_id}/" in normalized or f"/pdf/{book_id}/" in normalized:
+            return book_id
+    raise ValueError(f"Cannot infer book_id from source_path: {source_path}")
+
+
+def parse_source_path(source_path: str, book_id: str | None = None) -> SourceMetadata:
+    """Parse a checkpoint row source_path into structured metadata."""
+    resolved_book_id = book_id or infer_book_id(source_path)
+    filename = Path(source_path).name
+
+    if resolved_book_id == "oho":
+        match = OHO_FILENAME_RE.match(filename)
+        if not match:
+            raise ValueError(f"OHO filename does not match expected pattern: {filename}")
+        groups = match.groupdict()
+        return SourceMetadata(
+            book_id=resolved_book_id,
+            page_start=int(groups["page_start"]),
+            page_end=int(groups["page_end"]),
+            section_id=int(groups["section_id"]),
+            subsection_id=None,
+            batch_index=int(groups["batch_index"]),
+            source_filename=filename,
+        )
+
+    if resolved_book_id == "mor":
+        match = MOR_FILENAME_RE.match(filename)
+        if not match:
+            raise ValueError(f"MOR filename does not match expected pattern: {filename}")
+        groups = match.groupdict()
+        subsection_id = groups["subsection_id"]
+        return SourceMetadata(
+            book_id=resolved_book_id,
+            page_start=int(groups["page_start"]),
+            page_end=int(groups["page_end"]),
+            section_id=int(subsection_id.split(".")[0]),
+            subsection_id=subsection_id,
+            batch_index=None,
+            source_filename=filename,
+        )
+
+    if resolved_book_id == "ado":
+          legacy_match = ADO_LEGACY_FILENAME_RE.match(filename)
+          if legacy_match:
+              groups = legacy_match.groupdict()
+              return SourceMetadata(
+                  book_id=resolved_book_id,
+                  page_start=int(groups["page_start"]),
+                  page_end=int(groups["page_end"]),
+                  section_id=None,
+                  subsection_id=None,
+                  batch_index=int(groups["batch_index"]),
+                  source_filename=filename,
+              )
+          
+    match = BATCH_FILENAME_RE.match(filename)
+    if not match:
+        raise ValueError(
+            f"{resolved_book_id.upper()} filename does not match expected pattern: {filename}"
+        )
+    groups = match.groupdict()
+    return SourceMetadata(
+        book_id=resolved_book_id,
+        page_start=int(groups["page_start"]),
+        page_end=int(groups["page_end"]),
+        section_id=None,
+        subsection_id=None,
+        batch_index=int(groups["batch_index"]),
+        source_filename=filename,
+    )
+
+
+def logical_unit_key(meta: SourceMetadata) -> str:
+    """Return a stable assembly key for grouping checkpoint rows."""
+    if meta.book_id == "mor":
+        return f"sub_{meta.subsection_id}"
+    if meta.book_id == "oho":
+        return f"sec_{meta.section_id}"
+    raise NotImplementedError(
+        "Loose-tier books assign logical_unit_id after book_stream section detection."
+    )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pg_psycopg.py

@@ -0,0 +1,193 @@
+"""psycopg2-backed PostgreSQL upload helpers."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Generator
+
+import numpy as np
+
+from .chunker import SemanticChunkRecord
+from .config import EDITION_LABEL, EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_VERSION
+from .pg_uploader import PostgresContext
+
+
+@contextmanager
+def postgres_connection(dsn: str) -> Generator:
+    import psycopg2
+    from pgvector.psycopg2 import register_vector
+
+    connection = psycopg2.connect(dsn)
+    register_vector(connection)
+    try:
+        yield connection
+        connection.commit()
+    except Exception:
+        connection.rollback()
+        raise
+    finally:
+        connection.close()
+
+
+def build_context_with_connection(dsn: str, book_ids: list[str]) -> tuple[dict[str, str], str]:
+    edition_ids: dict[str, str] = {}
+    model_id: str | None = None
+
+    with postgres_connection(dsn) as conn:
+        with conn.cursor() as cur:
+            for book_id in book_ids:
+                cur.execute(
+                    """
+                    SELECT ce.edition_id
+                    FROM knowledge.corpus_edition ce
+                    INNER JOIN knowledge.corpus_source cs
+                        ON cs.corpus_source_id = ce.corpus_source_id
+                    WHERE cs.book_id = %s AND ce.status = 'active'
+                    LIMIT 1
+                    """,
+                    (book_id,),
+                )
+                row = cur.fetchone()
+                if row:
+                    edition_ids[book_id] = str(row[0])
+                    continue
+
+                cur.execute(
+                    "SELECT corpus_source_id FROM knowledge.corpus_source WHERE book_id = %s",
+                    (book_id,),
+                )
+                source_row = cur.fetchone()
+                if not source_row:
+                    raise RuntimeError(f"Missing corpus_source seed row for book_id={book_id}")
+                cur.execute(
+                    """
+                    INSERT INTO knowledge.corpus_edition (
+                        corpus_source_id, edition_label, status
+                    ) VALUES (%s, %s, 'active')
+                    RETURNING edition_id
+                    """,
+                    (source_row[0], EDITION_LABEL),
+                )
+                edition_ids[book_id] = str(cur.fetchone()[0])
+
+            cur.execute(
+                """
+                SELECT model_id FROM knowledge.embedding_model
+                WHERE model_name = %s AND model_version = %s
+                LIMIT 1
+                """,
+                (EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_VERSION),
+            )
+            model_row = cur.fetchone()
+            if not model_row:
+                raise RuntimeError("Missing embedding_model seed row for EmbeddingGemma")
+            model_id = str(model_row[0])
+
+    return edition_ids, model_id
+
+
+def upload_with_connection(
+    dsn: str,
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None,
+) -> None:
+    edition_id = context.edition_ids[record.book_id]
+    with postgres_connection(dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO knowledge.logical_unit (
+                    edition_id, unit_key, parent_title, page_start, page_end
+                ) VALUES (%s, %s, %s, %s, %s)
+                ON CONFLICT (edition_id, unit_key) DO UPDATE SET
+                    parent_title = EXCLUDED.parent_title,
+                    page_start = EXCLUDED.page_start,
+                    page_end = EXCLUDED.page_end
+                RETURNING logical_unit_id
+                """,
+                (
+                    edition_id,
+                    record.logical_unit_id,
+                    record.parent_title,
+                    record.page_start,
+                    record.page_end,
+                ),
+            )
+            logical_unit_uuid = cur.fetchone()[0]
+
+            cur.execute(
+                """
+                INSERT INTO knowledge.semantic_chunk (
+                    chunk_id, logical_unit_id, edition_id, chunker_version,
+                    chunk_index, content, token_count, content_hash,
+                    section_id, subsection_id, parent_title, page_start, page_end,
+                    is_active
+                ) VALUES (
+                    %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s,
+                    %s, %s, %s, %s, %s, true
+                )
+                ON CONFLICT (logical_unit_id, chunk_index, chunker_version) DO UPDATE SET
+                    content = EXCLUDED.content,
+                    token_count = EXCLUDED.token_count,
+                    content_hash = EXCLUDED.content_hash,
+                    section_id = EXCLUDED.section_id,
+                    subsection_id = EXCLUDED.subsection_id,
+                    parent_title = EXCLUDED.parent_title,
+                    page_start = EXCLUDED.page_start,
+                    page_end = EXCLUDED.page_end,
+                    is_active = true
+                RETURNING chunk_id
+                """,
+                (
+                    record.chunk_uuid,
+                    logical_unit_uuid,
+                    edition_id,
+                    record.chunker_version,
+                    record.chunk_index,
+                    record.content,
+                    record.token_count,
+                    record.content_hash,
+                    record.section_id,
+                    record.subsection_id,
+                    record.parent_title,
+                    record.page_start,
+                    record.page_end,
+                ),
+            )
+            chunk_id = cur.fetchone()[0]
+
+            for extraction_id in record.source_extraction_ids:
+                cur.execute(
+                    """
+                    INSERT INTO knowledge.chunk_provenance (
+                        chunk_id, checkpoint_db, processed_chunk_id, source_path
+                    ) VALUES (%s::uuid, %s, %s, %s)
+                    ON CONFLICT (chunk_id, checkpoint_db, processed_chunk_id) DO UPDATE SET
+                        source_path = EXCLUDED.source_path
+                    """,
+                    (
+                        chunk_id,
+                        checkpoint_db,
+                        extraction_id,
+                        (source_paths or {}).get(
+                            extraction_id,
+                            f"processed_chunks:{extraction_id}",
+                        ),
+                    ),
+                )
+
+            cur.execute(
+                """
+                INSERT INTO knowledge.semantic_chunk_embedding (
+                    chunk_id, model_id, edition_id, embedding, embedding_status, embedded_at
+                ) VALUES (%s::uuid, %s::uuid, %s::uuid, %s, 'indexed', now())
+                ON CONFLICT (chunk_id, model_id) DO UPDATE SET
+                    embedding = EXCLUDED.embedding,
+                    embedding_status = 'indexed',
+                    embedded_at = now()
+                """,
+                (chunk_id, context.model_id, edition_id, vector.tolist()),
+            )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pg_uploader.py

@@ -0,0 +1,288 @@
+"""Direct PostgreSQL uploader for the knowledge schema."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from .chunker import SemanticChunkRecord
+from .config import (
+    EDITION_LABEL,
+    EMBEDDING_DIMENSIONS,
+    EMBEDDING_MODEL_NAME,
+    EMBEDDING_MODEL_VERSION,
+)
+
+logger = logging.getLogger(__name__)
+
+SUPABASE_DIR = Path(__file__).resolve().parents[1] / "supabase"
+
+
+@dataclass(frozen=True)
+class PostgresContext:
+    """Resolved identifiers for PostgreSQL upserts."""
+
+    edition_ids: dict[str, str]
+    model_id: str
+    mode: str
+    dsn: str | None = None
+
+
+def resolve_database_dsn() -> str:
+    """Resolve a PostgreSQL connection string from environment variables."""
+    for key in ("SUPABASE_DB_URL", "DATABASE_URL", "SUPABASE_DATABASE_URL"):
+        value = os.getenv(key)
+        if value:
+            return value
+    raise RuntimeError(
+        "No PostgreSQL DSN found. Set SUPABASE_DB_URL/DATABASE_URL or use "
+        "Supabase CLI linked-project mode."
+    )
+
+
+def _supabase_cli_available() -> bool:
+    project_ref = SUPABASE_DIR / ".temp" / "project-ref"
+    try:
+        subprocess.run(
+            ["supabase", "--version"],
+            capture_output=True,
+            check=True,
+            text=True,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return False
+    return project_ref.exists()
+
+
+def _run_sql_via_cli(sql: str) -> dict[str, Any]:
+    """Execute SQL against the linked remote Supabase database."""
+    result = subprocess.run(
+        ["supabase", "db", "query", "--linked", sql],
+        cwd=SUPABASE_DIR,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    if not result.stdout.strip():
+        return {}
+    return json.loads(result.stdout)
+
+
+def _sql_literal(value: str | None) -> str:
+    if value is None:
+        return "NULL"
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _query_rows(sql: str) -> list[dict[str, Any]]:
+    payload = _run_sql_via_cli(sql)
+    return payload.get("rows") or []
+
+
+def _query_scalar(sql: str) -> str | None:
+    rows = _query_rows(sql)
+    if not rows:
+        return None
+    return str(next(iter(rows[0].values())))
+
+
+def build_postgres_context(
+    book_ids: list[str],
+    dsn: str | None = None,
+) -> PostgresContext:
+    """Resolve edition and embedding model ids for upload."""
+    use_cli = dsn is None and _supabase_cli_available()
+    if not use_cli:
+        resolved_dsn = dsn or resolve_database_dsn()
+        return _build_context_with_psycopg(resolved_dsn, book_ids)
+
+    edition_ids: dict[str, str] = {}
+    for book_id in book_ids:
+        edition_id = _query_scalar(
+            f"""
+            SELECT ce.edition_id::text
+            FROM knowledge.corpus_edition ce
+            INNER JOIN knowledge.corpus_source cs
+                ON cs.corpus_source_id = ce.corpus_source_id
+            WHERE cs.book_id = {_sql_literal(book_id)} AND ce.status = 'active'
+            LIMIT 1
+            """
+        )
+        if not edition_id:
+            corpus_source_id = _query_scalar(
+                "SELECT corpus_source_id::text "
+                f"FROM knowledge.corpus_source WHERE book_id = {_sql_literal(book_id)}"
+            )
+            if not corpus_source_id:
+                raise RuntimeError(f"Missing corpus_source seed row for book_id={book_id}")
+            edition_id = _query_scalar(
+                f"""
+                INSERT INTO knowledge.corpus_edition (
+                    corpus_source_id, edition_label, status
+                ) VALUES ('{corpus_source_id}', {_sql_literal(EDITION_LABEL)}, 'active')
+                RETURNING edition_id::text
+                """
+            )
+        if not edition_id:
+            raise RuntimeError(f"Unable to resolve active edition for book_id={book_id}")
+        edition_ids[book_id] = edition_id
+
+    model_id = _query_scalar(
+        f"""
+        SELECT model_id::text FROM knowledge.embedding_model
+        WHERE model_name = {_sql_literal(EMBEDDING_MODEL_NAME)}
+          AND model_version = {_sql_literal(EMBEDDING_MODEL_VERSION)}
+        LIMIT 1
+        """
+    )
+    if not model_id:
+        raise RuntimeError("Missing embedding_model seed row for EmbeddingGemma")
+    return PostgresContext(edition_ids=edition_ids, model_id=model_id, mode="cli")
+
+
+def _build_context_with_psycopg(dsn: str, book_ids: list[str]) -> PostgresContext:
+    from .pg_psycopg import build_context_with_connection
+
+    edition_ids, model_id = build_context_with_connection(dsn, book_ids)
+    return PostgresContext(
+        edition_ids=edition_ids,
+        model_id=model_id,
+        mode="psycopg",
+        dsn=dsn,
+    )
+
+
+def upload_semantic_chunk_pg(
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    """Upsert one chunk and embedding to Supabase PostgreSQL."""
+    if vector.shape[0] != EMBEDDING_DIMENSIONS:
+        raise ValueError(
+            f"Expected {EMBEDDING_DIMENSIONS}-d embedding, got {vector.shape[0]}"
+        )
+
+    if context.mode == "cli":
+        _upload_via_cli(context, record, vector, checkpoint_db, source_paths)
+        return
+
+    from .pg_psycopg import upload_with_connection
+
+    if not context.dsn:
+        raise RuntimeError("Missing PostgreSQL DSN for psycopg upload mode.")
+    upload_with_connection(
+        context.dsn,
+        context,
+        record,
+        vector,
+        checkpoint_db,
+        source_paths,
+    )
+
+
+def _upload_via_cli(
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None,
+) -> None:
+    edition_id = context.edition_ids[record.book_id]
+    vector_literal = json.dumps([float(v) for v in vector.tolist()])
+    provenance_values = []
+    for extraction_id in record.source_extraction_ids:
+        source_path = (source_paths or {}).get(
+            extraction_id,
+            f"processed_chunks:{extraction_id}",
+        )
+        provenance_values.append(
+            f"({_sql_literal(checkpoint_db)}, {extraction_id}, {_sql_literal(source_path)})"
+        )
+
+    sql = f"""
+    WITH upsert_unit AS (
+        INSERT INTO knowledge.logical_unit (
+            edition_id, unit_key, parent_title, page_start, page_end
+        ) VALUES (
+            '{edition_id}'::uuid,
+            {_sql_literal(record.logical_unit_id)},
+            {_sql_literal(record.parent_title)},
+            {record.page_start if record.page_start is not None else 'NULL'},
+            {record.page_end if record.page_end is not None else 'NULL'}
+        )
+        ON CONFLICT (edition_id, unit_key) DO UPDATE SET
+            parent_title = EXCLUDED.parent_title,
+            page_start = EXCLUDED.page_start,
+            page_end = EXCLUDED.page_end
+        RETURNING logical_unit_id
+    ),
+    upsert_chunk AS (
+        INSERT INTO knowledge.semantic_chunk (
+            chunk_id, logical_unit_id, edition_id, chunker_version,
+            chunk_index, content, token_count, content_hash,
+            section_id, subsection_id, parent_title, page_start, page_end,
+            is_active
+        )
+        SELECT
+            '{record.chunk_uuid}'::uuid,
+            logical_unit_id,
+            '{edition_id}'::uuid,
+            {_sql_literal(record.chunker_version)},
+            {record.chunk_index},
+            {_sql_literal(record.content)},
+            {record.token_count},
+            {_sql_literal(record.content_hash)},
+            {record.section_id if record.section_id is not None else 'NULL'},
+            {_sql_literal(record.subsection_id)},
+            {_sql_literal(record.parent_title)},
+            {record.page_start if record.page_start is not None else 'NULL'},
+            {record.page_end if record.page_end is not None else 'NULL'},
+            true
+        FROM upsert_unit
+        ON CONFLICT (logical_unit_id, chunk_index, chunker_version) DO UPDATE SET
+            content = EXCLUDED.content,
+            token_count = EXCLUDED.token_count,
+            content_hash = EXCLUDED.content_hash,
+            section_id = EXCLUDED.section_id,
+            subsection_id = EXCLUDED.subsection_id,
+            parent_title = EXCLUDED.parent_title,
+            page_start = EXCLUDED.page_start,
+            page_end = EXCLUDED.page_end,
+            is_active = true
+        RETURNING chunk_id
+    ),
+    upsert_provenance AS (
+        INSERT INTO knowledge.chunk_provenance (
+            chunk_id, checkpoint_db, processed_chunk_id, source_path
+        )
+        SELECT upsert_chunk.chunk_id, p.checkpoint_db, p.processed_chunk_id, p.source_path
+        FROM upsert_chunk
+        CROSS JOIN (VALUES {', '.join(provenance_values)}) AS p(
+            checkpoint_db, processed_chunk_id, source_path
+        )
+        ON CONFLICT (chunk_id, checkpoint_db, processed_chunk_id) DO UPDATE SET
+            source_path = EXCLUDED.source_path
+        RETURNING chunk_id
+    )
+    INSERT INTO knowledge.semantic_chunk_embedding (
+        chunk_id, model_id, edition_id, embedding, embedding_status, embedded_at
+    )
+    SELECT upsert_chunk.chunk_id, '{context.model_id}'::uuid, '{edition_id}'::uuid,
+           '{vector_literal}'::vector, 'indexed', now()
+    FROM upsert_chunk
+    ON CONFLICT (chunk_id, model_id) DO UPDATE SET
+        embedding = EXCLUDED.embedding,
+        embedding_status = 'indexed',
+        embedded_at = now();
+    """
+    _run_sql_via_cli(sql)

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pipeline.py

@@ -0,0 +1,277 @@
+"""End-to-end semantic chunking and Supabase embedding upload pipeline."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+from pathlib import Path
+
+from .assemble import assemble_logical_units
+from .chunker import chunk_logical_unit, embedding_text
+from .config import (
+    BOOK_DB_FILES,
+    CHUNKER_VERSION,
+    DEFAULT_CHUNKER_CONFIG,
+    DEFAULT_ENV_PATH,
+    resolve_corpus_db_path,
+)
+from .embedding import GemmaEmbedder, get_embedder
+from .sqlite_store import (
+    count_semantic_chunks,
+    finish_chunking_run,
+    init_semantic_tables,
+    load_pending_chunks,
+    load_processed_chunks,
+    mark_embedding_status,
+    start_chunking_run,
+    upsert_semantic_chunk,
+)
+from .supabase_uploader import build_supabase_context, upload_semantic_chunk
+
+try:
+    from .pg_uploader import build_postgres_context, upload_semantic_chunk_pg
+except ImportError:
+    build_postgres_context = None
+    upload_semantic_chunk_pg = None
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+async def run_chunk_stage(
+    book_id: str,
+    embedder: GemmaEmbedder,
+    force: bool = False,
+) -> tuple[int, int]:
+    """Chunk one book from processed_chunks into local semantic_chunks."""
+    db_path = resolve_corpus_db_path(book_id)
+    if not db_path.exists():
+        raise FileNotFoundError(f"Checkpoint database not found: {db_path}")
+
+    await init_semantic_tables(db_path)
+    run_id = await start_chunking_run(db_path, book_id, CHUNKER_VERSION)
+    units_processed = 0
+    chunks_written = 0
+
+    try:
+        processed_rows = await load_processed_chunks(db_path)
+        logical_units = assemble_logical_units(processed_rows, book_id=book_id)
+
+        # print("check the logical units", len(logical_units))
+        for unit in logical_units:
+            records = chunk_logical_unit(
+                unit,
+                embedder=embedder,
+                config=DEFAULT_CHUNKER_CONFIG,
+                chunker_version=CHUNKER_VERSION,
+            )
+            units_processed += 1
+            for record in records:
+                if not force:
+                    from .sqlite_store import chunk_exists
+
+                    exists = await chunk_exists(
+                        db_path,
+                        record.logical_unit_id,
+                        record.chunk_index,
+                        record.chunker_version,
+                        record.book_id,
+                    )
+                    if exists:
+                        continue
+                await upsert_semantic_chunk(db_path, record)
+                chunks_written += 1
+        # print("check the units processed and chunks written", units_processed, chunks_written)
+        await finish_chunking_run(
+            db_path,
+            run_id,
+            units_processed=units_processed,
+            chunks_written=chunks_written,
+        )
+        table_count = await count_semantic_chunks(db_path, book_id, CHUNKER_VERSION)
+        if chunks_written > 0 and chunks_written != table_count:
+            logger.warning(
+                "Chunk count mismatch for %s: chunks_written=%s semantic_chunks=%s "
+                "(possible logical_unit_id collision or stale rows; re-run with --force "
+                "after clearing old rows for this chunker_version)",
+                book_id,
+                chunks_written,
+                table_count,
+            )
+        logger.info(
+            "Chunk stage complete for %s: units=%s chunks=%s table_rows=%s",
+            book_id,
+            units_processed,
+            chunks_written,
+            table_count,
+        )
+        return units_processed, chunks_written
+    except Exception as exc:
+        await finish_chunking_run(
+            db_path,
+            run_id,
+            units_processed=units_processed,
+            chunks_written=chunks_written,
+            status="failed",
+            error_message=str(exc),
+        )
+        raise
+
+
+async def run_upload_stage(
+    book_id: str,
+    embedder: GemmaEmbedder,
+    env_path: Path = DEFAULT_ENV_PATH,
+) -> int:
+    """Embed pending semantic chunks and upload them to Supabase."""
+    db_path = resolve_corpus_db_path(book_id)
+    pending = await load_pending_chunks(db_path, book_id, CHUNKER_VERSION)
+    if not pending:
+        logger.info("No pending chunks to upload for %s", book_id)
+        return 0
+
+    processed_rows = await load_processed_chunks(db_path)
+    source_paths = {row_id: source_path for row_id, source_path, _ in processed_rows}
+    checkpoint_db = BOOK_DB_FILES[book_id]
+
+    from .supabase_uploader import load_supabase_env
+
+    load_supabase_env(env_path)
+
+    rest_context = None
+    pg_context = None
+
+    if build_postgres_context is not None:
+        try:
+            pg_context = build_postgres_context(book_ids=[book_id])
+            logger.info("Using PostgreSQL upload mode: %s", pg_context.mode)
+        except Exception as exc:
+            logger.warning("PostgreSQL upload unavailable: %s", exc)
+            pg_context = None
+
+    if pg_context is None:
+        try:
+            rest_context = build_supabase_context(book_ids=[book_id])
+            logger.info("Using Supabase REST upload")
+        except Exception as exc:
+            raise RuntimeError(
+                "Upload failed. Link Supabase CLI (`supabase link`) for PostgreSQL upload, "
+                "or expose the `knowledge` schema and provide SUPABASE_SERVICE_ROLE_KEY."
+            ) from exc
+
+    uploaded = 0
+    for record in pending:
+        try:
+            vector = embedder.embed_document(
+                embedding_text(record),
+                title=record.parent_title,
+            )
+            if pg_context is not None:
+                upload_semantic_chunk_pg(
+                    pg_context,
+                    record,
+                    vector,
+                    checkpoint_db=checkpoint_db,
+                    source_paths=source_paths,
+                )
+            else:
+                upload_semantic_chunk(
+                    rest_context,
+                    record,
+                    vector,
+                    checkpoint_db=checkpoint_db,
+                    source_paths=source_paths,
+                )
+            await mark_embedding_status(db_path, record.chunk_uuid, "done")
+            uploaded += 1
+            if uploaded % 10 == 0:
+                logger.info("Uploaded %s/%s chunks for %s", uploaded, len(pending), book_id)
+        except Exception as exc:
+            logger.exception(
+                "Failed to upload chunk %s (%s): %s",
+                record.chunk_uuid,
+                record.logical_unit_id,
+                exc,
+            )
+            await mark_embedding_status(db_path, record.chunk_uuid, "failed")
+
+    logger.info("Upload stage complete for %s: uploaded=%s", book_id, uploaded)
+    return uploaded
+
+
+async def run_pipeline(
+    book_ids: list[str],
+    chunk: bool = True,
+    upload: bool = True,
+    force: bool = False,
+    env_path: Path = DEFAULT_ENV_PATH,
+) -> None:
+    """Run chunk and/or upload stages for the requested books."""
+    embedder = get_embedder()
+
+    for book_id in book_ids:
+        logger.info("=== Processing book: %s ===", book_id)
+        if chunk:
+            await run_chunk_stage(book_id, embedder, force=force)
+        if upload:
+            await run_upload_stage(book_id, embedder, env_path=env_path)
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse CLI arguments for the ingestion pipeline."""
+    parser = argparse.ArgumentParser(
+        description="Semantic chunking and Supabase embedding upload pipeline.",
+    )
+    parser.add_argument(
+        "--books",
+        nargs="+",
+        choices=sorted(BOOK_DB_FILES.keys()),
+        default=["mor"],
+        help="Books to process (default: mor).",
+    )
+    parser.add_argument(
+        "--chunk-only",
+        action="store_true",
+        help="Run Stage 3 chunking only (SQLite staging).",
+    )
+    parser.add_argument(
+        "--upload-only",
+        action="store_true",
+        help="Run Stage 4 embedding upload only.",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Re-chunk even when chunk slots already exist.",
+    )
+    parser.add_argument(
+        "--env-path",
+        type=Path,
+        default=DEFAULT_ENV_PATH,
+        help="Path to .env file containing Supabase credentials.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """CLI entrypoint."""
+    args = parse_args()
+    chunk = not args.upload_only
+    upload = not args.chunk_only
+    asyncio.run(
+        run_pipeline(
+            book_ids=args.books,
+            chunk=chunk,
+            upload=upload,
+            force=args.force,
+            env_path=args.env_path,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/requirements.txt

@@ -0,0 +1,14 @@
+# Semantic chunking + Supabase embedding upload pipeline dependencies.
+# Install inside conda env `vkist_ultra` or an equivalent Python 3.11+ environment.
+
+aiosqlite>=0.20.0
+langchain-text-splitters>=0.3.0
+numpy>=1.26.0
+onnxruntime>=1.19.0
+optimum[onnxruntime]>=1.23.0
+python-dotenv>=1.0.0
+supabase>=2.10.0
+psycopg2-binary>=2.9.9
+pgvector>=0.3.0
+torch>=2.4.0
+transformers>=4.45.0

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/sqlite_store.py

@@ -0,0 +1,249 @@
+"""SQLite staging for semantic chunk checkpoint tables."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import aiosqlite
+
+from .chunker import SemanticChunkRecord
+
+logger = logging.getLogger(__name__)
+
+SEMANTIC_CHUNKS_DDL = """
+CREATE TABLE IF NOT EXISTS semantic_chunks (
+    id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+    chunk_uuid              TEXT NOT NULL UNIQUE,
+    logical_unit_id         TEXT NOT NULL,
+    chunk_index             INTEGER NOT NULL,
+    content                 TEXT NOT NULL,
+    token_count             INTEGER NOT NULL,
+    book_id                 TEXT NOT NULL,
+    section_id              INTEGER,
+    subsection_id           TEXT,
+    parent_title            TEXT,
+    page_start              INTEGER,
+    page_end                INTEGER,
+    source_extraction_ids   TEXT NOT NULL,
+    chunker_version         TEXT NOT NULL,
+    embedding_status        TEXT NOT NULL DEFAULT 'pending',
+    content_hash            TEXT NOT NULL,
+    created_at              TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (logical_unit_id, chunk_index, chunker_version, book_id)
+);
+"""
+
+CHUNKING_RUNS_DDL = """
+CREATE TABLE IF NOT EXISTS chunking_runs (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    book_id         TEXT NOT NULL,
+    chunker_version TEXT NOT NULL,
+    started_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    finished_at     TIMESTAMP,
+    status          TEXT NOT NULL DEFAULT 'running',
+    units_processed INTEGER DEFAULT 0,
+    chunks_written  INTEGER DEFAULT 0,
+    error_message   TEXT
+);
+"""
+
+
+async def init_semantic_tables(db_path: Path) -> None:
+    """Ensure semantic chunk staging tables exist in the checkpoint database."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(SEMANTIC_CHUNKS_DDL)
+        await db.execute(CHUNKING_RUNS_DDL)
+        await db.commit()
+
+
+async def load_processed_chunks(db_path: Path) -> list[tuple[int, str, str]]:
+    """Load all processed_chunks rows from a checkpoint database."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            "SELECT id, source_path, markdown_content FROM processed_chunks ORDER BY id"
+        )
+        return await cursor.fetchall()
+
+
+async def chunk_exists(
+    db_path: Path,
+    logical_unit_id: str,
+    chunk_index: int,
+    chunker_version: str,
+    book_id: str,
+) -> bool:
+    """Return True when an identical chunk slot already exists."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT 1 FROM semantic_chunks
+            WHERE logical_unit_id = ? AND chunk_index = ?
+              AND chunker_version = ? AND book_id = ?
+            LIMIT 1
+            """,
+            (logical_unit_id, chunk_index, chunker_version, book_id),
+        )
+        row = await cursor.fetchone()
+        return row is not None
+
+
+async def upsert_semantic_chunk(db_path: Path, record: SemanticChunkRecord) -> None:
+    """Insert or replace one semantic chunk row."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            INSERT INTO semantic_chunks (
+                chunk_uuid, logical_unit_id, chunk_index, content, token_count,
+                book_id, section_id, subsection_id, parent_title, page_start,
+                page_end, source_extraction_ids, chunker_version, embedding_status,
+                content_hash
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?)
+            ON CONFLICT(logical_unit_id, chunk_index, chunker_version, book_id)
+            DO UPDATE SET
+                chunk_uuid = excluded.chunk_uuid,
+                content = excluded.content,
+                token_count = excluded.token_count,
+                section_id = excluded.section_id,
+                subsection_id = excluded.subsection_id,
+                parent_title = excluded.parent_title,
+                page_start = excluded.page_start,
+                page_end = excluded.page_end,
+                source_extraction_ids = excluded.source_extraction_ids,
+                content_hash = excluded.content_hash,
+                embedding_status = 'pending'
+            """,
+            (
+                record.chunk_uuid,
+                record.logical_unit_id,
+                record.chunk_index,
+                record.content,
+                record.token_count,
+                record.book_id,
+                record.section_id,
+                record.subsection_id,
+                record.parent_title,
+                record.page_start,
+                record.page_end,
+                json.dumps(record.source_extraction_ids),
+                record.chunker_version,
+                record.content_hash,
+            ),
+        )
+        await db.commit()
+
+
+async def load_pending_chunks(
+    db_path: Path,
+    book_id: str,
+    chunker_version: str,
+) -> list[SemanticChunkRecord]:
+    """Load semantic chunks that still need embedding upload."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT chunk_uuid, logical_unit_id, chunk_index, content, token_count,
+                   book_id, section_id, subsection_id, parent_title, page_start,
+                   page_end, source_extraction_ids, chunker_version, content_hash
+            FROM semantic_chunks
+            WHERE book_id = ? AND chunker_version = ?
+              AND embedding_status IN ('pending', 'failed')
+            ORDER BY logical_unit_id, chunk_index
+            """,
+            (book_id, chunker_version),
+        )
+        rows = await cursor.fetchall()
+
+    records: list[SemanticChunkRecord] = []
+    for row in rows:
+        records.append(
+            SemanticChunkRecord(
+                chunk_uuid=row[0],
+                logical_unit_id=row[1],
+                chunk_index=row[2],
+                content=row[3],
+                token_count=row[4],
+                book_id=row[5],
+                section_id=row[6],
+                subsection_id=row[7],
+                parent_title=row[8],
+                page_start=row[9],
+                page_end=row[10],
+                source_extraction_ids=json.loads(row[11]),
+                chunker_version=row[12],
+                content_hash=row[13],
+            )
+        )
+    return records
+
+
+async def mark_embedding_status(
+    db_path: Path,
+    chunk_uuid: str,
+    status: str,
+) -> None:
+    """Update embedding_status for one staged chunk."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            "UPDATE semantic_chunks SET embedding_status = ? WHERE chunk_uuid = ?",
+            (status, chunk_uuid),
+        )
+        await db.commit()
+
+
+async def start_chunking_run(db_path: Path, book_id: str, chunker_version: str) -> int:
+    """Create a chunking run audit row and return its id."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            INSERT INTO chunking_runs (book_id, chunker_version, status)
+            VALUES (?, ?, 'running')
+            """,
+            (book_id, chunker_version),
+        )
+        await db.commit()
+        return cursor.lastrowid
+
+
+async def count_semantic_chunks(
+    db_path: Path,
+    book_id: str,
+    chunker_version: str,
+) -> int:
+    """Return the number of staged semantic chunks for one book and chunker version."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT COUNT(*) FROM semantic_chunks
+            WHERE book_id = ? AND chunker_version = ?
+            """,
+            (book_id, chunker_version),
+        )
+        row = await cursor.fetchone()
+        return int(row[0]) if row else 0
+
+
+async def finish_chunking_run(
+    db_path: Path,
+    run_id: int,
+    units_processed: int,
+    chunks_written: int,
+    status: str = "succeeded",
+    error_message: str | None = None,
+) -> None:
+    """Finalize a chunking run audit row."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            UPDATE chunking_runs
+            SET finished_at = CURRENT_TIMESTAMP,
+                units_processed = ?,
+                chunks_written = ?,
+                status = ?,
+                error_message = ?
+            WHERE id = ?
+            """,
+            (units_processed, chunks_written, status, error_message, run_id),
+        )
+        await db.commit()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/supabase_uploader.py

@@ -0,0 +1,306 @@
+"""Supabase upsert for semantic chunks and EmbeddingGemma vectors."""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from dotenv import load_dotenv
+from supabase import Client, create_client
+
+from .chunker import SemanticChunkRecord
+from .config import (
+    CHUNKER_VERSION,
+    DEFAULT_ENV_PATH,
+    EDITION_LABEL,
+    EMBEDDING_DIMENSIONS,
+    EMBEDDING_MODEL_NAME,
+    EMBEDDING_MODEL_VERSION,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SupabaseContext:
+    """Resolved Supabase identifiers required for ingestion upserts."""
+
+    client: Client
+    edition_ids: dict[str, str]
+    model_id: str
+
+
+def load_supabase_env(env_path: Path = DEFAULT_ENV_PATH) -> None:
+    """Load Supabase credentials from the project secrets file."""
+    if env_path.exists():
+        load_dotenv(env_path, override=False)
+
+
+def _resolve_supabase_key() -> str:
+    """Resolve the Supabase API key, preferring service role when available."""
+    service_role = os.getenv("SUPABASE_SERVICE_ROLE_KEY")
+    if service_role:
+        return service_role
+
+    publishable = os.getenv("SUPABASE_PUBLISHABLE_KEY")
+    if publishable:
+        logger.warning(
+            "Using SUPABASE_PUBLISHABLE_KEY for ingestion. "
+            "Writes require service_role or INSERT policies on knowledge.* tables."
+        )
+        return publishable
+
+    raise RuntimeError(
+        "Missing Supabase credentials. Set SUPABASE_URL and "
+        "SUPABASE_PUBLISHABLE_KEY (or SUPABASE_SERVICE_ROLE_KEY)."
+    )
+
+
+def create_supabase_client() -> Client:
+    """Create a Supabase client from environment variables."""
+    load_supabase_env()
+    url = os.getenv("SUPABASE_URL")
+    if not url:
+        raise RuntimeError("SUPABASE_URL is not set.")
+    key = _resolve_supabase_key()
+    return create_client(url, key)
+
+
+def _schema(client: Client):
+    return client.schema("knowledge")
+
+
+def _single_row(response: Any, label: str) -> dict[str, Any]:
+    data = response.data
+    if not data:
+        raise RuntimeError(f"No rows returned for {label}.")
+    return data[0]
+
+
+def _get_corpus_source_id(client: Client, book_id: str) -> str:
+    response = (
+        _schema(client)
+        .table("corpus_source")
+        .select("corpus_source_id")
+        .eq("book_id", book_id)
+        .limit(1)
+        .execute()
+    )
+    return _single_row(response, f"corpus_source[{book_id}]")["corpus_source_id"]
+
+
+def _get_embedding_model_id(client: Client) -> str:
+    response = (
+        _schema(client)
+        .table("embedding_model")
+        .select("model_id")
+        .eq("model_name", EMBEDDING_MODEL_NAME)
+        .eq("model_version", EMBEDDING_MODEL_VERSION)
+        .limit(1)
+        .execute()
+    )
+    return _single_row(response, "embedding_model")["model_id"]
+
+
+def ensure_active_edition(client: Client, book_id: str) -> str:
+    """Return the active edition_id for a book, creating one when missing."""
+    corpus_source_id = _get_corpus_source_id(client, book_id)
+    response = (
+        _schema(client)
+        .table("corpus_edition")
+        .select("edition_id")
+        .eq("corpus_source_id", corpus_source_id)
+        .eq("status", "active")
+        .limit(1)
+        .execute()
+    )
+    if response.data:
+        return response.data[0]["edition_id"]
+
+    insert_response = (
+        _schema(client)
+        .table("corpus_edition")
+        .insert(
+            {
+                "corpus_source_id": corpus_source_id,
+                "edition_label": EDITION_LABEL,
+                "status": "active",
+            }
+        )
+        .execute()
+    )
+    edition_id = _single_row(insert_response, f"corpus_edition[{book_id}]")["edition_id"]
+    logger.info("Created active corpus_edition for %s: %s", book_id, edition_id)
+    return edition_id
+
+
+def build_supabase_context(
+    client: Client | None = None,
+    book_ids: list[str] | None = None,
+) -> SupabaseContext:
+    """Resolve edition and model identifiers for a set of books."""
+    resolved_client = client or create_supabase_client()
+    target_books = book_ids or ["ado", "tny", "oho", "mor"]
+
+    try:
+        edition_ids = {
+            book_id: ensure_active_edition(resolved_client, book_id)
+            for book_id in target_books
+        }
+        model_id = _get_embedding_model_id(resolved_client)
+    except Exception as exc:
+        raise RuntimeError(
+            "Supabase REST upload failed. Ensure the hosted project exposes the "
+            "`knowledge` schema (Dashboard → API → Exposed schemas) and provide "
+            "SUPABASE_SERVICE_ROLE_KEY, or set SUPABASE_DB_URL for direct PostgreSQL upload."
+        ) from exc
+
+    return SupabaseContext(
+        client=resolved_client,
+        edition_ids=edition_ids,
+        model_id=model_id,
+    )
+
+
+def _upsert_logical_unit(
+    client: Client,
+    edition_id: str,
+    record: SemanticChunkRecord,
+) -> str:
+    payload = {
+        "edition_id": edition_id,
+        "unit_key": record.logical_unit_id,
+        "parent_title": record.parent_title,
+        "page_start": record.page_start,
+        "page_end": record.page_end,
+    }
+    response = (
+        _schema(client)
+        .table("logical_unit")
+        .upsert(payload, on_conflict="edition_id,unit_key")
+        .select("logical_unit_id")
+        .execute()
+    )
+    return _single_row(response, f"logical_unit[{record.logical_unit_id}]")["logical_unit_id"]
+
+
+def _upsert_semantic_chunk(
+    client: Client,
+    edition_id: str,
+    logical_unit_uuid: str,
+    record: SemanticChunkRecord,
+) -> str:
+    payload = {
+        "chunk_id": record.chunk_uuid,
+        "logical_unit_id": logical_unit_uuid,
+        "edition_id": edition_id,
+        "chunker_version": record.chunker_version,
+        "chunk_index": record.chunk_index,
+        "content": record.content,
+        "token_count": record.token_count,
+        "content_hash": record.content_hash,
+        "section_id": record.section_id,
+        "subsection_id": record.subsection_id,
+        "parent_title": record.parent_title,
+        "page_start": record.page_start,
+        "page_end": record.page_end,
+        "is_active": True,
+    }
+    response = (
+        _schema(client)
+        .table("semantic_chunk")
+        .upsert(payload, on_conflict="logical_unit_id,chunk_index,chunker_version")
+        .select("chunk_id")
+        .execute()
+    )
+    return _single_row(response, f"semantic_chunk[{record.chunk_uuid}]")["chunk_id"]
+
+
+def _upsert_chunk_provenance(
+    client: Client,
+    chunk_id: str,
+    checkpoint_db: str,
+    source_extraction_ids: list[int],
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    rows = []
+    for extraction_id in source_extraction_ids:
+        rows.append(
+            {
+                "chunk_id": chunk_id,
+                "checkpoint_db": checkpoint_db,
+                "processed_chunk_id": extraction_id,
+                "source_path": (source_paths or {}).get(
+                    extraction_id,
+                    f"processed_chunks:{extraction_id}",
+                ),
+            }
+        )
+    _schema(client).table("chunk_provenance").upsert(
+        rows,
+        on_conflict="chunk_id,checkpoint_db,processed_chunk_id",
+    ).execute()
+
+
+def _format_vector(vector: np.ndarray) -> list[float]:
+    if vector.shape[0] != EMBEDDING_DIMENSIONS:
+        raise ValueError(
+            f"Expected {EMBEDDING_DIMENSIONS}-d embedding, got {vector.shape[0]}"
+        )
+    return [float(value) for value in vector.tolist()]
+
+
+def _upsert_embedding(
+    client: Client,
+    chunk_id: str,
+    edition_id: str,
+    model_id: str,
+    vector: np.ndarray,
+) -> None:
+    payload = {
+        "chunk_id": chunk_id,
+        "model_id": model_id,
+        "edition_id": edition_id,
+        "embedding": _format_vector(vector),
+        "embedding_status": "indexed",
+    }
+    _schema(client).table("semantic_chunk_embedding").upsert(
+        payload,
+        on_conflict="chunk_id,model_id",
+    ).execute()
+
+
+def upload_semantic_chunk(
+    context: SupabaseContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    """Upsert one chunk and its embedding vector to Supabase."""
+    edition_id = context.edition_ids[record.book_id]
+    logical_unit_uuid = _upsert_logical_unit(context.client, edition_id, record)
+    chunk_id = _upsert_semantic_chunk(
+        context.client,
+        edition_id,
+        logical_unit_uuid,
+        record,
+    )
+    _upsert_chunk_provenance(
+        context.client,
+        chunk_id,
+        checkpoint_db,
+        record.source_extraction_ids,
+        source_paths=source_paths,
+    )
+    _upsert_embedding(
+        context.client,
+        chunk_id,
+        edition_id,
+        context.model_id,
+        vector,
+    )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/.gitignore

@@ -0,0 +1,8 @@
+# Supabase
+.branches
+.temp
+
+# dotenvx
+.env.keys
+.env.local
+.env.*.local

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/README.md

@@ -0,0 +1,108 @@
+# Knowledge Supabase — Semantic Vector DB
+
+PostgreSQL + pgvector schema for clinical textbook RAG, derived from [`spec/pg_semantic_vector_db/er_diagram.md`](../spec/pg_semantic_vector_db/er_diagram.md).
+
+## Layout
+
+```
+knowledge/supabase/
+├── config.toml
+├── README.md
+└── migrations/
+    ├── 20260705000000_semantic_vector_db_schema.sql   # tables + HNSW index
+    ├── 20260705000001_semantic_vector_db_seed.sql     # corpus_source, chunker, model
+    ├── 20260705000002_semantic_vector_db_rls_and_rpc.sql  # RLS + match_semantic_chunks()
+    ├── 20260705000003_expose_knowledge_api_schema.sql # PostgREST grants
+    ├── 20260705000004_chunker_profile_v2.sql          # header+semantic_v2 profile
+    ├── 20260705000005_corpus_bibliography.sql         # bibliography columns + contributors
+    └── 20260705000006_corpus_bibliography_seed.sql    # seed from corpus/source_metadata.md
+```
+
+All application tables live in the **`knowledge`** schema (not `public`).
+
+## Apply to hosted Supabase
+
+### Option A — Supabase CLI (recommended)
+
+```bash
+cd workspace/sprint_1_2/CODEBASE/knowledge/supabase
+
+# Link your project (once)
+supabase link --project-ref <your-project-ref>
+
+# Push migrations
+supabase db push
+```
+
+### Option B — SQL Editor
+
+Run each migration file in order in the Supabase Dashboard → SQL Editor.
+
+## Local development
+
+```bash
+cd workspace/sprint_1_2/CODEBASE/knowledge/supabase
+supabase start
+supabase db reset   # applies all migrations + seed
+```
+
+Local Postgres: `postgresql://postgres:postgres@localhost:54322/postgres`
+
+## Environment variables
+
+Add to your backend `.env`:
+
+```bash
+SUPABASE_URL=https://<project-ref>.supabase.co
+SUPABASE_SERVICE_ROLE_KEY=<service-role-key>   # ingestion + RAG backend only
+# Optional: for authenticated read policies
+SUPABASE_ANON_KEY=<anon-key>
+```
+
+**Never expose `SUPABASE_SERVICE_ROLE_KEY` to the frontend.**
+
+## Runtime RAG query
+
+After Stage 4 embed upsert, call the RPC from the backend:
+
+```sql
+SELECT * FROM knowledge.match_semantic_chunks(
+  query_embedding := $1::extensions.vector(768),
+  match_count     := 5,
+  filter_book_ids := ARRAY['mor', 'oho']
+);
+```
+
+Active-edition filter is enforced inside the function:
+
+- `corpus_edition.status = 'active'`
+- `semantic_chunk.is_active = true`
+- `semantic_chunk_embedding.embedding_status = 'indexed'`
+
+## Ingestion upsert flow (Stage 4)
+
+1. Ensure `corpus_edition` exists for the book (status `active`).
+2. Upsert `logical_unit` rows per assembled unit.
+3. Upsert `semantic_chunk` rows (idempotent on `logical_unit_id + chunk_index + chunker_version`).
+4. Insert `chunk_provenance` rows from SQLite `source_extraction_ids`.
+5. Upsert `semantic_chunk_embedding` with `embedding_status = 'indexed'` and the 768-d vector.
+
+## Verify after migration
+
+```sql
+SELECT book_id, full_title, subject_area FROM knowledge.corpus_source;
+SELECT book_id, edition_label, publisher_name, published_year
+FROM knowledge.v_corpus_citation;
+SELECT cs.book_id, cc.role, cc.full_name
+FROM knowledge.corpus_contributor cc
+JOIN knowledge.corpus_source cs ON cs.corpus_source_id = cc.corpus_source_id
+ORDER BY cs.book_id, cc.sort_order;
+SELECT chunker_version FROM knowledge.chunker_profile;
+SELECT model_name, dimensions FROM knowledge.embedding_model WHERE is_active;
+```
+
+## References
+
+- [`spec/pg_semantic_vector_db/er_diagram.md`](../spec/pg_semantic_vector_db/er_diagram.md)
+- [`spec/pg_semantic_vector_db/supabase_schema.md`](../spec/pg_semantic_vector_db/supabase_schema.md)
+- [`spec/ingestion/schema.md`](../spec/ingestion/schema.md) — SQLite staging → PG mapping

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/config.toml

@@ -0,0 +1,414 @@
+# For detailed configuration reference documentation, visit:
+# https://supabase.com/docs/guides/local-development/cli/config
+# A string used to distinguish different Supabase projects on the same host. Defaults to the
+# working directory name when running `supabase init`.
+project_id = "supabase"
+
+[api]
+enabled = true
+# Port to use for the API URL.
+port = 54321
+# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
+# endpoints. `public` and `graphql_public` schemas are included by default.
+schemas = ["public", "graphql_public", "knowledge"]
+# Extra schemas to add to the search_path of every request.
+extra_search_path = ["public", "extensions"]
+# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
+# for accidental or malicious requests.
+max_rows = 1000
+# Controls whether new tables, views, sequences and functions created in the `public` schema by
+# `postgres` are reachable through the Data API roles (`anon`, `authenticated`, `service_role`)
+# without explicit GRANTs. When unset, new entities are NOT auto-exposed, matching the new cloud
+# default. Set to `true` to keep the legacy behaviour of auto-exposing new entities; this is
+# deprecated and the field is removed on 2026-10-30 once the always-revoked behaviour is permanent.
+# auto_expose_new_tables = true
+
+[api.tls]
+# Enable HTTPS endpoints locally using a self-signed certificate.
+enabled = false
+# Paths to self-signed certificate pair.
+# cert_path = "../certs/my-cert.pem"
+# key_path = "../certs/my-key.pem"
+
+[db]
+# Port to use for the local database URL.
+port = 54322
+# Port used by db diff command to initialize the shadow database.
+shadow_port = 54320
+# Maximum amount of time to wait for health check when starting the local database.
+health_timeout = "2m"
+# The database major version to use. This has to be the same as your remote database's. Run `SHOW
+# server_version;` on the remote database to check.
+major_version = 17
+
+[db.pooler]
+enabled = false
+# Port to use for the local connection pooler.
+port = 54329
+# Specifies when a server connection can be reused by other clients.
+# Configure one of the supported pooler modes: `transaction`, `session`.
+pool_mode = "transaction"
+# How many server connections to allow per user/database pair.
+default_pool_size = 20
+# Maximum number of client connections allowed.
+max_client_conn = 100
+
+# [db.vault]
+# secret_key = "env(SECRET_VALUE)"
+
+[db.migrations]
+# If disabled, migrations will be skipped during a db push or reset.
+enabled = true
+# Specifies an ordered list of schema files that describe your database.
+# Supports glob patterns relative to supabase directory: "./schemas/*.sql"
+schema_paths = []
+
+[db.seed]
+# If enabled, seeds the database after migrations during a db reset.
+enabled = true
+# Specifies an ordered list of seed files to load during db reset.
+# Supports glob patterns relative to supabase directory: "./seeds/*.sql"
+sql_paths = ["./seed.sql"]
+
+[db.network_restrictions]
+# Enable management of network restrictions.
+enabled = false
+# List of IPv4 CIDR blocks allowed to connect to the database.
+# Defaults to allow all IPv4 connections. Set empty array to block all IPs.
+allowed_cidrs = ["0.0.0.0/0"]
+# List of IPv6 CIDR blocks allowed to connect to the database.
+# Defaults to allow all IPv6 connections. Set empty array to block all IPs.
+allowed_cidrs_v6 = ["::/0"]
+
+# Uncomment to reject non-secure connections to the database.
+# [db.ssl_enforcement]
+# enabled = true
+
+[realtime]
+enabled = true
+# Bind realtime via either IPv4 or IPv6. (default: IPv4)
+# ip_version = "IPv6"
+# The maximum length in bytes of HTTP request headers. (default: 4096)
+# max_header_length = 4096
+
+[studio]
+enabled = true
+# Port to use for Supabase Studio.
+port = 54323
+# External URL of the API server that frontend connects to.
+api_url = "http://127.0.0.1"
+# OpenAI API Key to use for Supabase AI in the Supabase Studio.
+openai_api_key = "env(OPENAI_API_KEY)"
+
+# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
+# are monitored, and you can view the emails that would have been sent from the web interface.
+[local_smtp]
+enabled = true
+# Port to use for the email testing server web interface.
+port = 54324
+# Uncomment to expose additional ports for testing user applications that send emails.
+# smtp_port = 54325
+# pop3_port = 54326
+# admin_email = "admin@email.com"
+# sender_name = "Admin"
+
+[storage]
+enabled = true
+# The maximum file size allowed (e.g. "5MB", "500KB").
+file_size_limit = "50MiB"
+
+# Uncomment to configure local storage buckets
+# [storage.buckets.images]
+# public = false
+# file_size_limit = "50MiB"
+# allowed_mime_types = ["image/png", "image/jpeg"]
+# objects_path = "./images"
+
+# Allow connections via S3 compatible clients
+[storage.s3_protocol]
+enabled = true
+
+# Image transformation API is available to Supabase Pro plan.
+# [storage.image_transformation]
+# enabled = true
+
+# Store analytical data in S3 for running ETL jobs over Iceberg Catalog
+# This feature is only available on the hosted platform.
+[storage.analytics]
+enabled = false
+max_namespaces = 5
+max_tables = 10
+max_catalogs = 2
+
+# Analytics Buckets is available to Supabase Pro plan.
+# [storage.analytics.buckets.my-warehouse]
+
+# Store vector embeddings in S3 for large and durable datasets
+[storage.vector]
+enabled = true
+max_buckets = 10
+max_indexes = 5
+
+# Vector Buckets is available to Supabase Pro plan.
+# [storage.vector.buckets.documents-openai]
+
+[auth]
+enabled = true
+# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
+# in emails.
+site_url = "http://127.0.0.1:3000"
+# The public URL that Auth serves on. Defaults to the API external URL with `/auth/v1` appended.
+# external_url = ""
+# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
+additional_redirect_urls = ["https://127.0.0.1:3000"]
+# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week).
+jwt_expiry = 3600
+# JWT issuer URL. If not set, defaults to auth.external_url.
+# jwt_issuer = ""
+# Path to JWT signing key. DO NOT commit your signing keys file to git.
+# signing_keys_path = "./signing_keys.json"
+# If disabled, the refresh token will never expire.
+enable_refresh_token_rotation = true
+# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds.
+# Requires enable_refresh_token_rotation = true.
+refresh_token_reuse_interval = 10
+# Allow/disallow new user signups to your project.
+enable_signup = true
+# Allow/disallow anonymous sign-ins to your project.
+enable_anonymous_sign_ins = false
+# Allow/disallow testing manual linking of accounts
+enable_manual_linking = false
+# Passwords shorter than this value will be rejected as weak. Minimum 6, recommended 8 or more.
+minimum_password_length = 6
+# Passwords that do not meet the following requirements will be rejected as weak. Supported values
+# are: `letters_digits`, `lower_upper_letters_digits`, `lower_upper_letters_digits_symbols`
+password_requirements = ""
+
+# Configure passkey sign-ins.
+# [auth.passkey]
+# enabled = false
+
+# Configure WebAuthn relying party settings (required when passkey is enabled).
+# [auth.webauthn]
+# rp_display_name = "Supabase"
+# rp_id = "localhost"
+# rp_origins = ["http://127.0.0.1:3000"]
+
+[auth.rate_limit]
+# Number of emails that can be sent per hour. Requires auth.email.smtp to be enabled.
+email_sent = 2
+# Number of SMS messages that can be sent per hour. Requires auth.sms to be enabled.
+sms_sent = 30
+# Number of anonymous sign-ins that can be made per hour per IP address. Requires enable_anonymous_sign_ins = true.
+anonymous_users = 30
+# Number of sessions that can be refreshed in a 5 minute interval per IP address.
+token_refresh = 150
+# Number of sign up and sign-in requests that can be made in a 5 minute interval per IP address (excludes anonymous users).
+sign_in_sign_ups = 30
+# Number of OTP / Magic link verifications that can be made in a 5 minute interval per IP address.
+token_verifications = 30
+# Number of Web3 logins that can be made in a 5 minute interval per IP address.
+web3 = 30
+
+# Configure one of the supported captcha providers: `hcaptcha`, `turnstile`.
+# [auth.captcha]
+# enabled = true
+# provider = "hcaptcha"
+# secret = ""
+
+[auth.email]
+# Allow/disallow new user signups via email to your project.
+enable_signup = true
+# If enabled, a user will be required to confirm any email change on both the old, and new email
+# addresses. If disabled, only the new email is required to confirm.
+double_confirm_changes = true
+# If enabled, users need to confirm their email address before signing in.
+enable_confirmations = false
+# If enabled, users will need to reauthenticate or have logged in recently to change their password.
+secure_password_change = false
+# Controls the minimum amount of time that must pass before sending another signup confirmation or password reset email.
+max_frequency = "1s"
+# Number of characters used in the email OTP.
+otp_length = 6
+# Number of seconds before the email OTP expires (defaults to 1 hour).
+otp_expiry = 3600
+
+# Use a production-ready SMTP server
+# [auth.email.smtp]
+# enabled = true
+# host = "smtp.sendgrid.net"
+# port = 587
+# user = "apikey"
+# pass = "env(SENDGRID_API_KEY)"
+# admin_email = "admin@email.com"
+# sender_name = "Admin"
+
+# Uncomment to customize email template
+# [auth.email.template.invite]
+# subject = "You have been invited"
+# content_path = "./supabase/templates/invite.html"
+
+# Uncomment to customize notification email template
+# [auth.email.notification.password_changed]
+# enabled = true
+# subject = "Your password has been changed"
+# content_path = "./templates/password_changed_notification.html"
+
+[auth.sms]
+# Allow/disallow new user signups via SMS to your project.
+enable_signup = false
+# If enabled, users need to confirm their phone number before signing in.
+enable_confirmations = false
+# Template for sending OTP to users
+template = "Your code is {{ `{{ .Code }}` }}"
+# Controls the minimum amount of time that must pass before sending another sms otp.
+max_frequency = "5s"
+
+# Use pre-defined map of phone number to OTP for testing.
+# [auth.sms.test_otp]
+# 4152127777 = "123456"
+
+# Configure logged in session timeouts.
+# [auth.sessions]
+# Force log out after the specified duration.
+# timebox = "24h"
+# Force log out if the user has been inactive longer than the specified duration.
+# inactivity_timeout = "8h"
+
+# This hook runs before a new user is created and allows developers to reject the request based on the incoming user object.
+# [auth.hook.before_user_created]
+# enabled = true
+# uri = "pg-functions://postgres/auth/before-user-created-hook"
+
+# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used.
+# [auth.hook.custom_access_token]
+# enabled = true
+# uri = "pg-functions://<database>/<schema>/<hook_name>"
+
+# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`.
+[auth.sms.twilio]
+enabled = false
+account_sid = ""
+message_service_sid = ""
+# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead:
+auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)"
+
+# Multi-factor-authentication is available to Supabase Pro plan.
+[auth.mfa]
+# Control how many MFA factors can be enrolled at once per user.
+max_enrolled_factors = 10
+
+# Control MFA via App Authenticator (TOTP)
+[auth.mfa.totp]
+enroll_enabled = false
+verify_enabled = false
+
+# Configure MFA via Phone Messaging
+[auth.mfa.phone]
+enroll_enabled = false
+verify_enabled = false
+otp_length = 6
+template = "Your code is {{ `{{ .Code }}` }}"
+max_frequency = "5s"
+
+# Configure MFA via WebAuthn
+# [auth.mfa.web_authn]
+# enroll_enabled = true
+# verify_enabled = true
+
+# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
+# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`,
+# `twitter`, `x`, `slack`, `spotify`, `workos`, `zoom`.
+[auth.external.apple]
+enabled = false
+client_id = ""
+# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead:
+secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)"
+# Overrides the default auth callback URL derived from auth.external_url.
+redirect_uri = ""
+# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure,
+# or any other third-party OIDC providers.
+url = ""
+# If enabled, the nonce check will be skipped. Required for local sign in with Google auth.
+skip_nonce_check = false
+# If enabled, it will allow the user to successfully authenticate when the provider does not return an email address.
+email_optional = false
+
+# Allow Solana wallet holders to sign in to your project via the Sign in with Solana (SIWS, EIP-4361) standard.
+# You can configure "web3" rate limit in the [auth.rate_limit] section and set up [auth.captcha] if self-hosting.
+[auth.web3.solana]
+enabled = false
+
+# Use Firebase Auth as a third-party provider alongside Supabase Auth.
+[auth.third_party.firebase]
+enabled = false
+# project_id = "my-firebase-project"
+
+# Use Auth0 as a third-party provider alongside Supabase Auth.
+[auth.third_party.auth0]
+enabled = false
+# tenant = "my-auth0-tenant"
+# tenant_region = "us"
+
+# Use AWS Cognito (Amplify) as a third-party provider alongside Supabase Auth.
+[auth.third_party.aws_cognito]
+enabled = false
+# user_pool_id = "my-user-pool-id"
+# user_pool_region = "us-east-1"
+
+# Use Clerk as a third-party provider alongside Supabase Auth.
+[auth.third_party.clerk]
+enabled = false
+# Obtain from https://clerk.com/setup/supabase
+# domain = "example.clerk.accounts.dev"
+
+# OAuth server configuration
+[auth.oauth_server]
+# Enable OAuth server functionality
+enabled = false
+# Path for OAuth consent flow UI
+authorization_url_path = "/oauth/consent"
+# Allow dynamic client registration
+allow_dynamic_registration = false
+
+[edge_runtime]
+enabled = true
+# Supported request policies: `oneshot`, `per_worker`.
+# `per_worker` (default) — enables hot reload during local development.
+# `oneshot` — fallback mode if hot reload causes issues (e.g. in large repos or with symlinks).
+policy = "per_worker"
+# Port to attach the Chrome inspector for debugging edge functions.
+inspector_port = 8083
+# The Deno major version to use.
+deno_version = 2
+
+# [edge_runtime.secrets]
+# secret_key = "env(SECRET_VALUE)"
+
+[analytics]
+enabled = true
+port = 54327
+# Configure one of the supported backends: `postgres`, `bigquery`.
+backend = "postgres"
+
+# Experimental features may be deprecated any time
+[experimental]
+# Configures Postgres storage engine to use OrioleDB (S3)
+orioledb_version = ""
+# Configures S3 bucket URL, eg. <bucket_name>.s3-<region>.amazonaws.com
+s3_host = "env(S3_HOST)"
+# Configures S3 bucket region, eg. us-east-1
+s3_region = "env(S3_REGION)"
+# Configures AWS_ACCESS_KEY_ID for S3 bucket
+s3_access_key = "env(S3_ACCESS_KEY)"
+# Configures AWS_SECRET_ACCESS_KEY for S3 bucket
+s3_secret_key = "env(S3_SECRET_KEY)"
+
+# pg-delta is the schema diff engine for db diff / db pull / db remote commit.
+# Set enabled = false to fall back to the legacy migra engine.
+[experimental.pgdelta]
+enabled = true
+# Directory under `supabase/` where declarative files are written.
+# declarative_schema_path = "./database"
+# JSON string passed through to pg-delta SQL formatting.
+# format_options = "{\"keywordCase\":\"upper\",\"indent\":2,\"maxWidth\":80,\"commaStyle\":\"trailing\"}"

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000000_semantic_vector_db_schema.sql

@@ -0,0 +1,300 @@
+-- pg-Semantic-Vector-DB schema for Supabase
+-- Source: knowledge/spec/pg_semantic_vector_db/er_diagram.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Extensions & schema
+-- ---------------------------------------------------------------------------
+CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public;
+CREATE SCHEMA IF NOT EXISTS knowledge;
+
+COMMENT ON SCHEMA knowledge IS
+  'Clinical textbook semantic chunks and EmbeddingGemma vectors for RAG retrieval.';
+
+-- ---------------------------------------------------------------------------
+-- corpus_source
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.corpus_source (
+    corpus_source_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    book_id          text NOT NULL,
+    display_name     text NOT NULL,
+    constraint_tier  text NOT NULL,
+    locale           text NOT NULL DEFAULT 'vi-VN',
+    created_at       timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT corpus_source_book_id_key UNIQUE (book_id),
+    CONSTRAINT corpus_source_constraint_tier_check
+        CHECK (constraint_tier IN ('loose', 'section', 'subsection'))
+);
+
+COMMENT ON TABLE knowledge.corpus_source IS
+  'Registry of ingestible textbook corpora (ADO, TNY, OHO, MOR).';
+
+-- ---------------------------------------------------------------------------
+-- corpus_edition
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.corpus_edition (
+    edition_id         uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    corpus_source_id   uuid NOT NULL REFERENCES knowledge.corpus_source (corpus_source_id)
+        ON DELETE RESTRICT,
+    edition_label      text NOT NULL,
+    source_pdf_sha256  text,
+    status             text NOT NULL DEFAULT 'draft',
+    effective_from     date,
+    effective_to       date,
+    created_at         timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT corpus_edition_status_check
+        CHECK (status IN ('draft', 'active', 'superseded', 'archived'))
+);
+
+CREATE UNIQUE INDEX uq_corpus_edition_one_active
+    ON knowledge.corpus_edition (corpus_source_id)
+    WHERE status = 'active';
+
+CREATE INDEX idx_corpus_edition_source_status
+    ON knowledge.corpus_edition (corpus_source_id, status);
+
+COMMENT ON TABLE knowledge.corpus_edition IS
+  'Versioned PDF snapshot; re-chunking and re-embedding are scoped to an edition.';
+
+-- ---------------------------------------------------------------------------
+-- chunker_profile
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.chunker_profile (
+    chunker_version text PRIMARY KEY,
+    description     text,
+    max_tokens      integer NOT NULL DEFAULT 800,
+    min_tokens      integer NOT NULL DEFAULT 120,
+    overlap_tokens  integer NOT NULL DEFAULT 64,
+    params_json     jsonb NOT NULL DEFAULT '{}'::jsonb,
+    created_at      timestamptz NOT NULL DEFAULT now()
+);
+
+COMMENT ON TABLE knowledge.chunker_profile IS
+  'Version registry for header + semantic chunking algorithm parameters.';
+
+-- ---------------------------------------------------------------------------
+-- embedding_model
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.embedding_model (
+    model_id       uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    model_name     text NOT NULL,
+    model_version  text NOT NULL,
+    dimensions     integer NOT NULL,
+    purpose        text NOT NULL DEFAULT 'rag',
+    is_active      boolean NOT NULL DEFAULT true,
+    registered_at  timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT embedding_model_purpose_check
+        CHECK (purpose IN ('rag')),
+    CONSTRAINT embedding_model_dimensions_check
+        CHECK (dimensions > 0),
+    CONSTRAINT embedding_model_name_version_key
+        UNIQUE (model_name, model_version)
+);
+
+COMMENT ON TABLE knowledge.embedding_model IS
+  'Registry of embedding models allowed to populate the RAG vector index.';
+
+-- ---------------------------------------------------------------------------
+-- structure_node (optional TOC tree)
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.structure_node (
+    structure_node_id        uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id               uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    parent_structure_node_id uuid REFERENCES knowledge.structure_node (structure_node_id)
+        ON DELETE CASCADE,
+    node_type                text NOT NULL,
+    node_key                 text NOT NULL,
+    title                    text,
+    page_start               integer,
+    page_end                 integer,
+    sort_order               integer NOT NULL DEFAULT 0,
+
+    CONSTRAINT structure_node_node_type_check
+        CHECK (node_type IN ('part', 'section', 'subsection')),
+    CONSTRAINT structure_node_edition_node_key_key
+        UNIQUE (edition_id, node_key)
+);
+
+CREATE INDEX idx_structure_node_edition_parent
+    ON knowledge.structure_node (edition_id, parent_structure_node_id);
+
+COMMENT ON TABLE knowledge.structure_node IS
+  'Optional document hierarchy (TOC); strongly populated for OHO/MOR.';
+
+-- ---------------------------------------------------------------------------
+-- logical_unit
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.logical_unit (
+    logical_unit_id   uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    structure_node_id uuid REFERENCES knowledge.structure_node (structure_node_id)
+        ON DELETE SET NULL,
+    unit_key          text NOT NULL,
+    parent_title      text,
+    page_start        integer,
+    page_end          integer,
+    assembled_at      timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT logical_unit_edition_unit_key_key
+        UNIQUE (edition_id, unit_key)
+);
+
+CREATE INDEX idx_logical_unit_edition
+    ON knowledge.logical_unit (edition_id);
+
+COMMENT ON TABLE knowledge.logical_unit IS
+  'Assembly boundary before chunking (sec_N, sub_X.Y, etc.).';
+
+-- ---------------------------------------------------------------------------
+-- semantic_chunk
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.semantic_chunk (
+    chunk_id         uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    logical_unit_id  uuid NOT NULL REFERENCES knowledge.logical_unit (logical_unit_id)
+        ON DELETE CASCADE,
+    edition_id       uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    chunker_version  text NOT NULL REFERENCES knowledge.chunker_profile (chunker_version)
+        ON DELETE RESTRICT,
+    chunk_index      integer NOT NULL,
+    content          text NOT NULL,
+    token_count      integer NOT NULL,
+    content_hash     text NOT NULL,
+    section_id       integer,
+    subsection_id    text,
+    parent_title     text,
+    page_start       integer,
+    page_end         integer,
+    is_active        boolean NOT NULL DEFAULT true,
+    created_at       timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT semantic_chunk_logical_unit_chunk_index_version_key
+        UNIQUE (logical_unit_id, chunk_index, chunker_version),
+    CONSTRAINT semantic_chunk_token_count_check
+        CHECK (token_count >= 0)
+);
+
+CREATE INDEX idx_semantic_chunk_edition_active
+    ON knowledge.semantic_chunk (edition_id, is_active);
+
+CREATE INDEX idx_semantic_chunk_chunker_version
+    ON knowledge.semantic_chunk (chunker_version);
+
+COMMENT ON TABLE knowledge.semantic_chunk IS
+  'Retrieval-ready text unit with citation metadata; chunk_id is the API citation key.';
+
+-- ---------------------------------------------------------------------------
+-- chunk_provenance
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.chunk_provenance (
+    provenance_id       uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id            uuid NOT NULL REFERENCES knowledge.semantic_chunk (chunk_id)
+        ON DELETE CASCADE,
+    checkpoint_db       text NOT NULL,
+    processed_chunk_id  bigint NOT NULL,
+    source_path         text NOT NULL,
+
+    CONSTRAINT chunk_provenance_chunk_checkpoint_source_key
+        UNIQUE (chunk_id, checkpoint_db, processed_chunk_id)
+);
+
+CREATE INDEX idx_chunk_provenance_checkpoint
+    ON knowledge.chunk_provenance (checkpoint_db, processed_chunk_id);
+
+COMMENT ON TABLE knowledge.chunk_provenance IS
+  'Traceability from semantic chunks back to SQLite processed_chunks rows.';
+
+-- ---------------------------------------------------------------------------
+-- semantic_chunk_embedding
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.semantic_chunk_embedding (
+    embedding_id      uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id          uuid NOT NULL REFERENCES knowledge.semantic_chunk (chunk_id)
+        ON DELETE CASCADE,
+    model_id          uuid NOT NULL REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE RESTRICT,
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    embedding         public.vector(768),
+    embedding_status  text NOT NULL DEFAULT 'pending',
+    embedded_at       timestamptz,
+
+    CONSTRAINT semantic_chunk_embedding_chunk_model_key
+        UNIQUE (chunk_id, model_id),
+    CONSTRAINT semantic_chunk_embedding_status_check
+        CHECK (embedding_status IN ('pending', 'indexed', 'failed'))
+);
+
+CREATE INDEX idx_semantic_chunk_embedding_edition_status
+    ON knowledge.semantic_chunk_embedding (edition_id, embedding_status);
+
+-- HNSW index for cosine similarity search (PoC scale ~15K vectors)
+CREATE INDEX idx_semantic_chunk_embedding_hnsw
+    ON knowledge.semantic_chunk_embedding
+    USING hnsw (embedding public.vector_cosine_ops)
+    WITH (m = 16, ef_construction = 64);
+
+COMMENT ON TABLE knowledge.semantic_chunk_embedding IS
+  'EmbeddingGemma vectors; separated from semantic_chunk for re-embed without duplicating text.';
+
+-- ---------------------------------------------------------------------------
+-- ingestion_run
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.ingestion_run (
+    run_id              uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id          uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    chunker_version     text NOT NULL REFERENCES knowledge.chunker_profile (chunker_version)
+        ON DELETE RESTRICT,
+    model_id            uuid REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE SET NULL,
+    stage               text NOT NULL,
+    status              text NOT NULL DEFAULT 'running',
+    units_processed     integer NOT NULL DEFAULT 0,
+    chunks_written      integer NOT NULL DEFAULT 0,
+    embeddings_written  integer NOT NULL DEFAULT 0,
+    error_message       text,
+    started_at          timestamptz NOT NULL DEFAULT now(),
+    finished_at         timestamptz,
+
+    CONSTRAINT ingestion_run_stage_check
+        CHECK (stage IN ('chunk', 'embed', 'full')),
+    CONSTRAINT ingestion_run_status_check
+        CHECK (status IN ('running', 'succeeded', 'failed'))
+);
+
+CREATE INDEX idx_ingestion_run_edition_started
+    ON knowledge.ingestion_run (edition_id, started_at DESC);
+
+COMMENT ON TABLE knowledge.ingestion_run IS
+  'Audit log for chunk and embed pipeline executions.';
+
+-- ---------------------------------------------------------------------------
+-- vector_index_snapshot
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.vector_index_snapshot (
+    index_snapshot_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    model_id          uuid NOT NULL REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE RESTRICT,
+    index_name        text NOT NULL,
+    hnsw_params       jsonb NOT NULL DEFAULT '{}'::jsonb,
+    is_active         boolean NOT NULL DEFAULT false,
+    built_at          timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE UNIQUE INDEX uq_vector_index_snapshot_one_active
+    ON knowledge.vector_index_snapshot (edition_id, model_id)
+    WHERE is_active = true;
+
+COMMENT ON TABLE knowledge.vector_index_snapshot IS
+  'HNSW index rebuild lifecycle for blue/green reindex.';
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000001_semantic_vector_db_seed.sql

@@ -0,0 +1,55 @@
+-- Seed reference data for pg-Semantic-Vector-DB
+-- Source: knowledge/spec/ingestion/corpus_profiles.md, semantic_chunking_spec.md
+
+BEGIN;
+
+INSERT INTO knowledge.corpus_source (book_id, display_name, constraint_tier, locale)
+VALUES
+    ('ado', 'ADO Clinical Textbook', 'loose', 'vi-VN'),
+    ('tny', 'TNY Clinical Textbook', 'loose', 'vi-VN'),
+    ('oho', 'OHO Clinical Textbook', 'section', 'vi-VN'),
+    ('mor', 'MOR Clinical Textbook', 'subsection', 'vi-VN')
+ON CONFLICT (book_id) DO UPDATE
+SET
+    display_name    = EXCLUDED.display_name,
+    constraint_tier = EXCLUDED.constraint_tier,
+    locale          = EXCLUDED.locale;
+
+INSERT INTO knowledge.chunker_profile (
+    chunker_version,
+    description,
+    max_tokens,
+    min_tokens,
+    overlap_tokens,
+    params_json
+)
+VALUES (
+    'header+semantic_v1',
+    'Header pre-split + embedding breakpoint semantic split on oversized pieces',
+    800,
+    120,
+    64,
+    jsonb_build_object(
+        'breakpoint_threshold_type', 'percentile',
+        'breakpoint_threshold_amount', 95,
+        'embedding_model', 'EmbeddingGemma',
+        'protected_blocks', jsonb_build_array('table', 'code_fence', 'short_list')
+    )
+)
+ON CONFLICT (chunker_version) DO UPDATE
+SET
+    description    = EXCLUDED.description,
+    max_tokens     = EXCLUDED.max_tokens,
+    min_tokens     = EXCLUDED.min_tokens,
+    overlap_tokens = EXCLUDED.overlap_tokens,
+    params_json    = EXCLUDED.params_json;
+
+INSERT INTO knowledge.embedding_model (model_name, model_version, dimensions, purpose, is_active)
+VALUES ('EmbeddingGemma', '1', 768, 'rag', true)
+ON CONFLICT (model_name, model_version) DO UPDATE
+SET
+    dimensions = EXCLUDED.dimensions,
+    purpose    = EXCLUDED.purpose,
+    is_active  = EXCLUDED.is_active;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000002_semantic_vector_db_rls_and_rpc.sql

@@ -0,0 +1,119 @@
+-- RLS policies and RAG search RPC for pg-Semantic-Vector-DB on Supabase
+-- Backend ingestion uses service_role (bypasses RLS).
+-- anon/authenticated have no direct access by default.
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Grants (service_role has full access; authenticated denied via RLS)
+-- ---------------------------------------------------------------------------
+GRANT USAGE ON SCHEMA knowledge TO postgres, service_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA knowledge TO service_role;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA knowledge TO service_role;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA knowledge
+    GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO service_role;
+
+-- ---------------------------------------------------------------------------
+-- Row Level Security
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_source ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.corpus_edition ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.structure_node ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.logical_unit ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.chunker_profile ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.semantic_chunk ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.chunk_provenance ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.embedding_model ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.semantic_chunk_embedding ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.ingestion_run ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.vector_index_snapshot ENABLE ROW LEVEL SECURITY;
+
+-- Read-only policy for authenticated backend JWT role (optional future use).
+-- Replace with a dedicated `knowledge_reader` role if you split backend auth.
+DROP POLICY IF EXISTS semantic_chunk_read_active ON knowledge.semantic_chunk;
+CREATE POLICY semantic_chunk_read_active
+    ON knowledge.semantic_chunk
+    FOR SELECT
+    TO authenticated
+    USING (is_active = true);
+
+DROP POLICY IF EXISTS semantic_chunk_embedding_read_indexed ON knowledge.semantic_chunk_embedding;
+CREATE POLICY semantic_chunk_embedding_read_indexed
+    ON knowledge.semantic_chunk_embedding
+    FOR SELECT
+    TO authenticated
+    USING (embedding_status = 'indexed');
+
+-- ---------------------------------------------------------------------------
+-- RAG similarity search RPC
+-- ---------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION knowledge.match_semantic_chunks(
+    query_embedding public.vector(768),
+    match_count integer DEFAULT 5,
+    filter_book_ids text[] DEFAULT NULL,
+    filter_edition_ids uuid[] DEFAULT NULL
+)
+RETURNS TABLE (
+    chunk_id uuid,
+    content text,
+    parent_title text,
+    page_start integer,
+    page_end integer,
+    section_id integer,
+    subsection_id text,
+    book_id text,
+    edition_id uuid,
+    chunk_index integer,
+    chunker_version text,
+    similarity double precision
+)
+LANGUAGE sql
+STABLE
+SECURITY DEFINER
+SET search_path = knowledge, extensions, public
+AS $$
+    SELECT
+        sc.chunk_id,
+        sc.content,
+        sc.parent_title,
+        sc.page_start,
+        sc.page_end,
+        sc.section_id,
+        sc.subsection_id,
+        cs.book_id,
+        sc.edition_id,
+        sc.chunk_index,
+        sc.chunker_version,
+        1 - (sce.embedding <=> query_embedding) AS similarity
+    FROM knowledge.semantic_chunk_embedding sce
+    INNER JOIN knowledge.semantic_chunk sc
+        ON sc.chunk_id = sce.chunk_id
+    INNER JOIN knowledge.corpus_edition ce
+        ON ce.edition_id = sc.edition_id
+    INNER JOIN knowledge.corpus_source cs
+        ON cs.corpus_source_id = ce.corpus_source_id
+    WHERE ce.status = 'active'
+      AND sc.is_active = true
+      AND sce.embedding_status = 'indexed'
+      AND sce.embedding IS NOT NULL
+      AND (
+          filter_book_ids IS NULL
+          OR cs.book_id = ANY (filter_book_ids)
+      )
+      AND (
+          filter_edition_ids IS NULL
+          OR sc.edition_id = ANY (filter_edition_ids)
+      )
+    ORDER BY sce.embedding <=> query_embedding
+    LIMIT GREATEST(match_count, 1);
+$$;
+
+COMMENT ON FUNCTION knowledge.match_semantic_chunks IS
+  'Top-k cosine similarity search over active-edition semantic chunks. '
+  'Call from backend RAG coordinator with service_role or authenticated JWT.';
+
+GRANT EXECUTE ON FUNCTION knowledge.match_semantic_chunks TO service_role, authenticated;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000003_expose_knowledge_api_schema.sql

@@ -0,0 +1,14 @@
+-- Expose knowledge schema to PostgREST roles used by Supabase Data API.
+-- Hosted projects must also add `knowledge` under Dashboard → Project Settings → API → Exposed schemas.
+
+BEGIN;
+
+GRANT USAGE ON SCHEMA knowledge TO anon, authenticated, service_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA knowledge TO service_role;
+GRANT SELECT ON ALL TABLES IN SCHEMA knowledge TO authenticated;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA knowledge TO service_role;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA knowledge
+    GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO service_role;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000004_chunker_profile_v2.sql

@@ -0,0 +1,35 @@
+-- Register header+semantic_v2 chunker profile (prose/table split before semantic chunking)
+
+BEGIN;
+
+INSERT INTO knowledge.chunker_profile (
+    chunker_version,
+    description,
+    max_tokens,
+    min_tokens,
+    overlap_tokens,
+    params_json
+)
+VALUES (
+    'header+semantic_v2',
+    'Header pre-split + prose/table block extraction + semantic split on oversized prose',
+    800,
+    120,
+    64,
+    jsonb_build_object(
+        'breakpoint_threshold_type', 'percentile',
+        'breakpoint_threshold_amount', 95,
+        'embedding_model', 'EmbeddingGemma',
+        'protected_blocks', jsonb_build_array('table', 'code_fence', 'short_list'),
+        'table_split', true
+    )
+)
+ON CONFLICT (chunker_version) DO UPDATE
+SET
+    description    = EXCLUDED.description,
+    max_tokens     = EXCLUDED.max_tokens,
+    min_tokens     = EXCLUDED.min_tokens,
+    overlap_tokens = EXCLUDED.overlap_tokens,
+    params_json    = EXCLUDED.params_json;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000005_corpus_bibliography.sql

@@ -0,0 +1,139 @@
+-- Bibliographic metadata for corpus_source, corpus_edition, and contributors.
+-- Source: knowledge/corpus/source_metadata.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Work-level bibliographic fields (corpus_source)
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_source
+    ADD COLUMN IF NOT EXISTS full_title       text,
+    ADD COLUMN IF NOT EXISTS short_title      text,
+    ADD COLUMN IF NOT EXISTS subtitle         text,
+    ADD COLUMN IF NOT EXISTS primary_language text NOT NULL DEFAULT 'vi',
+    ADD COLUMN IF NOT EXISTS subject_area     text,
+    ADD COLUMN IF NOT EXISTS bibliography_json jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+COMMENT ON COLUMN knowledge.corpus_source.full_title IS
+  'Official bibliographic title for citations (may differ from display_name).';
+COMMENT ON COLUMN knowledge.corpus_source.bibliography_json IS
+  'Extensible work-level metadata (resource_type, keywords, series, etc.).';
+
+UPDATE knowledge.corpus_source
+SET full_title = display_name
+WHERE full_title IS NULL;
+
+-- ---------------------------------------------------------------------------
+-- Edition-level publication metadata (corpus_edition)
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_edition
+    ADD COLUMN IF NOT EXISTS publisher_name    text,
+    ADD COLUMN IF NOT EXISTS publisher_place   text,
+    ADD COLUMN IF NOT EXISTS published_date    date,
+    ADD COLUMN IF NOT EXISTS published_year    smallint,
+    ADD COLUMN IF NOT EXISTS isbn_13           text,
+    ADD COLUMN IF NOT EXISTS isbn_10           text,
+    ADD COLUMN IF NOT EXISTS edition_number    text,
+    ADD COLUMN IF NOT EXISTS volume_label      text,
+    ADD COLUMN IF NOT EXISTS source_uri        text,
+    ADD COLUMN IF NOT EXISTS bibliography_json  jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+COMMENT ON COLUMN knowledge.corpus_edition.published_date IS
+  'Exact publication date when known; otherwise use published_year only.';
+COMMENT ON COLUMN knowledge.corpus_edition.source_uri IS
+  'Canonical source file or URI for this edition (PDF, markdown, etc.).';
+
+CREATE INDEX IF NOT EXISTS idx_corpus_edition_published_year
+    ON knowledge.corpus_edition (published_year);
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_corpus_edition_isbn_13
+    ON knowledge.corpus_edition (isbn_13)
+    WHERE isbn_13 IS NOT NULL;
+
+-- ---------------------------------------------------------------------------
+-- Contributors (authors, editors, translators)
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS knowledge.corpus_contributor (
+    contributor_id     uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    corpus_source_id   uuid NOT NULL
+        REFERENCES knowledge.corpus_source (corpus_source_id)
+        ON DELETE CASCADE,
+    edition_id         uuid
+        REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    role               text NOT NULL DEFAULT 'author',
+    full_name          text NOT NULL,
+    name_romanized     text,
+    affiliation        text,
+    orcid              text,
+    sort_order         integer NOT NULL DEFAULT 0,
+    created_at         timestamptz NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT corpus_contributor_role_check
+        CHECK (role IN ('author', 'editor', 'translator', 'compiler', 'institution'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_corpus_contributor_source
+    ON knowledge.corpus_contributor (corpus_source_id, sort_order);
+
+CREATE INDEX IF NOT EXISTS idx_corpus_contributor_edition
+    ON knowledge.corpus_contributor (edition_id, sort_order)
+    WHERE edition_id IS NOT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_corpus_contributor_work_level
+    ON knowledge.corpus_contributor (corpus_source_id, role, full_name)
+    WHERE edition_id IS NULL;
+
+COMMENT ON TABLE knowledge.corpus_contributor IS
+  'People or institutions credited for a corpus work (edition_id NULL) '
+  'or a specific edition.';
+
+-- ---------------------------------------------------------------------------
+-- Citation view for RAG / UI (active edition + aggregated authors)
+-- ---------------------------------------------------------------------------
+CREATE OR REPLACE VIEW knowledge.v_corpus_citation AS
+SELECT
+    cs.corpus_source_id,
+    cs.book_id,
+    cs.display_name,
+    cs.full_title,
+    cs.short_title,
+    cs.subtitle,
+    cs.primary_language,
+    cs.subject_area,
+    cs.constraint_tier,
+    ce.edition_id,
+    ce.edition_label,
+    ce.edition_number,
+    ce.publisher_name,
+    ce.publisher_place,
+    ce.published_date,
+    ce.published_year,
+    ce.isbn_13,
+    ce.isbn_10,
+    ce.source_uri,
+    ce.status AS edition_status,
+    (
+        SELECT string_agg(cc.full_name, '; ' ORDER BY cc.sort_order, cc.full_name)
+        FROM knowledge.corpus_contributor cc
+        WHERE cc.corpus_source_id = cs.corpus_source_id
+          AND cc.role IN ('author', 'editor')
+          AND cc.edition_id IS NULL
+    ) AS contributors
+FROM knowledge.corpus_source cs
+INNER JOIN knowledge.corpus_edition ce
+    ON ce.corpus_source_id = cs.corpus_source_id
+WHERE ce.status = 'active';
+
+COMMENT ON VIEW knowledge.v_corpus_citation IS
+  'Citation bundle for active corpus editions (NFR-18 grounding).';
+
+-- ---------------------------------------------------------------------------
+-- RLS + grants for new table
+-- ---------------------------------------------------------------------------
+GRANT SELECT, INSERT, UPDATE, DELETE ON knowledge.corpus_contributor TO service_role;
+ALTER TABLE knowledge.corpus_contributor ENABLE ROW LEVEL SECURITY;
+
+GRANT SELECT ON knowledge.v_corpus_citation TO service_role, authenticated;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000006_corpus_bibliography_seed.sql

@@ -0,0 +1,368 @@
+-- Seed bibliographic metadata from knowledge/corpus/source_metadata.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- ADO — Dictionary of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Dictionary of Rheumatology',
+    full_title       = 'Dictionary of Rheumatology',
+    short_title      = 'Dictionary of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Musculoskeletal Medicine',
+    bibliography_json = '{
+        "resource_type": "medical_dictionary",
+        "discipline": "rheumatology",
+        "series": null,
+        "keywords": [
+            "rheumatology",
+            "musculoskeletal disorders",
+            "immunology",
+            "osteoporosis",
+            "clinical terminology"
+        ]
+    }'::jsonb
+WHERE book_id = 'ado';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = '2009 English Edition',
+    edition_number    = NULL,
+    publisher_name    = 'Springer-Verlag/Wien',
+    publisher_place   = 'Vienna, Austria',
+    published_year    = 2009,
+    published_date    = NULL,
+    isbn_13           = '9783211685846',
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'ADO.pdf',
+    bibliography_json = '{
+        "library_of_congress_control_number": "2008940520",
+        "imprint": "SpringerWienNewYork",
+        "resource_type": "dictionary"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'ado'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, status,
+    publisher_name, publisher_place, published_year,
+    isbn_13, source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    '2009 English Edition',
+    'active',
+    'Springer-Verlag/Wien',
+    'Vienna, Austria',
+    2009,
+    '9783211685846',
+    'ADO.pdf',
+    '{
+        "library_of_congress_control_number": "2008940520",
+        "imprint": "SpringerWienNewYork",
+        "resource_type": "dictionary"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'ado'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'ado'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('editor', 'Jozef Rovenský', 1),
+        ('editor', 'Juraj Payer', 2),
+        ('author', 'Roy B. Clague', 3),
+        ('author', 'Manfred Herold', 4),
+        ('author', 'Milan Bayer', 5),
+        ('author', 'Helena Tauchmannová', 6),
+        ('author', 'Miroslav Ferenčík', 7),
+        ('author', 'Zdenko Killinger', 8)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'ado';
+
+-- ---------------------------------------------------------------------------
+-- MOR — Manual of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Manual of Rheumatology',
+    full_title       = 'Manual of Rheumatology',
+    short_title      = 'Manual of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Clinical Immunology',
+    bibliography_json = '{
+        "resource_type": "clinical_manual",
+        "discipline": "rheumatology",
+        "keywords": [
+            "rheumatology",
+            "clinical immunology",
+            "arthritis",
+            "musculoskeletal disease",
+            "clinical practice"
+        ]
+    }'::jsonb
+WHERE book_id = 'mor';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'Sixth Edition',
+    edition_number    = '6',
+    publisher_name    = 'CBS Publishers & Distributors Pvt. Ltd.',
+    publisher_place   = 'New Delhi, India',
+    published_year    = 2024,
+    published_date    = NULL,
+    isbn_13           = NULL,
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'MOR.pdf',
+    bibliography_json = '{
+        "format": "ebook",
+        "edition_type": "sixth",
+        "resource_type": "clinical_manual"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'mor'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    publisher_name, publisher_place, published_year,
+    source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'Sixth Edition',
+    '6',
+    'active',
+    'CBS Publishers & Distributors Pvt. Ltd.',
+    'New Delhi, India',
+    2024,
+    'MOR.pdf',
+    '{
+        "format": "ebook",
+        "edition_type": "sixth",
+        "resource_type": "clinical_manual"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'mor'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'mor'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('editor', 'Ramnath Misra', 1),
+        ('editor', 'Sakir Ahmed', 2),
+        ('editor', 'Prasanta Padhan', 3),
+        ('editor', 'Molly Mary Thabah', 4)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'mor';
+
+-- ---------------------------------------------------------------------------
+-- OHO — Oxford Handbook of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Oxford Handbook of Rheumatology',
+    full_title       = 'Oxford Handbook of Rheumatology',
+    short_title      = 'Oxford Handbook of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Musculoskeletal Medicine',
+    bibliography_json = '{
+        "resource_type": "handbook",
+        "series": "Oxford Medical Publications",
+        "keywords": [
+            "rheumatology",
+            "musculoskeletal medicine",
+            "clinical handbook",
+            "arthritis",
+            "autoimmune disease"
+        ]
+    }'::jsonb
+WHERE book_id = 'oho';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'Fourth Edition',
+    edition_number    = '4',
+    publisher_name    = 'Oxford University Press',
+    publisher_place   = 'Oxford, United Kingdom',
+    published_year    = 2018,
+    published_date    = NULL,
+    isbn_13           = '9780198728252',
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'OHO.pdf',
+    bibliography_json = '{
+        "series": "Oxford Medical Publications",
+        "library_of_congress_control_number": "2017960672",
+        "ebook_isbn": "9780191043949",
+        "resource_type": "clinical_handbook"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'oho'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    publisher_name, publisher_place, published_year,
+    isbn_13, source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'Fourth Edition',
+    '4',
+    'active',
+    'Oxford University Press',
+    'Oxford, United Kingdom',
+    2018,
+    '9780198728252',
+    'OHO.pdf',
+    '{
+        "series": "Oxford Medical Publications",
+        "library_of_congress_control_number": "2017960672",
+        "ebook_isbn": "9780191043949",
+        "resource_type": "clinical_handbook"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'oho'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'oho'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('author', 'Gavin Clunie', 1),
+        ('author', 'Nick Wilkinson', 2),
+        ('author', 'Elena Nikiphorou', 3),
+        ('author', 'Deepak Jadon', 4)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'oho';
+
+-- ---------------------------------------------------------------------------
+-- TNY — Thuật Ngữ Y Khoa: Cơ Xương Khớp
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    full_title       = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    short_title      = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    subtitle         = 'Thuật Ngữ Tiếng Anh Y Khoa Cho Người Mới Bắt Đầu',
+    primary_language = 'vi',
+    subject_area     = 'Musculoskeletal Medicine; Medical Terminology',
+    bibliography_json = '{
+        "resource_type": "educational_material",
+        "series": null,
+        "keywords": [
+            "thuật ngữ y khoa",
+            "cơ xương khớp",
+            "tiếng anh y khoa",
+            "giải phẫu",
+            "bệnh học"
+        ]
+    }'::jsonb
+WHERE book_id = 'tny';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'First Edition',
+    edition_number    = '1',
+    publisher_name    = NULL,
+    publisher_place   = NULL,
+    published_year    = NULL,
+    published_date    = NULL,
+    isbn_13           = NULL,
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'tny.md',
+    bibliography_json = '{
+        "series": null,
+        "library_of_congress_control_number": null,
+        "ebook_isbn": null,
+        "resource_type": "handout_notes"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'tny'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'First Edition',
+    '1',
+    'active',
+    'tny.md',
+    '{
+        "series": null,
+        "library_of_congress_control_number": null,
+        "ebook_isbn": null,
+        "resource_type": "handout_notes"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'tny'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'tny'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, 'author', 'Nguyễn Thái Duy', 1
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'tny';
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/source_query/source_query.sql

@@ -0,0 +1,24 @@
+-- SELECT
+--   sc.chunk_id,
+--   sc.content,
+--   cs.book_id,
+--   cs.display_name AS corpus_display_name,
+--   ce.edition_label,
+--   ce.status AS edition_status,
+--   sc.parent_title,
+--   sc.section_id,
+--   sc.subsection_id,
+--   sc.page_start,
+--   sc.page_end,
+--   sc.chunk_index,
+--   sc.chunker_version
+-- FROM knowledge.semantic_chunk sc
+-- JOIN knowledge.corpus_edition ce
+--   ON ce.edition_id = sc.edition_id
+-- JOIN knowledge.corpus_source cs
+--   ON cs.corpus_source_id = ce.corpus_source_id
+-- WHERE cs.book_id = 'mor' -- other option include: 'oho', 'ado', 'tny'
+--   AND ce.status = 'active'
+--   AND sc.is_active = true
+-- ORDER BY sc.chunk_index
+-- LIMIT 5;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/upload_supabase.sh

@@ -0,0 +1,15 @@
+PYTHONPATH=. python -m implementation.ingestion.pipeline \
+    --books ado mor oho \
+    --upload-only \
+    --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books ado --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books mor --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books oho --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env

workspace/sprint_1_2/CODEBASE/knowledge/spec/ingestion/corpus_profiles.md

@@ -0,0 +1,263 @@
+# Corpus Profiles — Book-Specific Ingestion Rules
+
+## Purpose
+
+Defines per-textbook metadata parsing, logical-unit assembly keys, and boundary sources for the semantic chunking pipeline.
+
+## Profile Summary
+
+| Book | `book_id` | Constraint tier | Physical unit | Logical unit | Assembly |
+|------|-----------|-----------------|---------------|--------------|----------|
+| ADO | `ado` | Loose | 5 pages, unconstrained | Section (via TOC/headers) | Stitch all batches → detect boundaries |
+| TNY | `tny` | Loose | 5 pages, unconstrained | Section (via TOC/headers) | Stitch all batches → detect boundaries |
+| OHO | `oho` | Section | 5 pages within section | Section | Group by `section_id`, stitch batches |
+| MOR | `mor` | Subsection | Whole subsection (var. pages) | Subsection | One row = one unit (no stitch) |
+
+---
+
+## ADO
+
+### Physical split
+
+- Pattern: same as TNY — fixed 5-page batches across entire bounded PDF
+- Output: `corpus/pdf/ado/batches/`
+- Filename: `batch-{NN}_chunk-{PPP}-{PPP}.pdf`
+- Example: `batch-02_chunk-006-010.pdf`
+
+### Filename regex
+
+```regex
+^batch-(?P<batch_index>\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$
+```
+
+### Logical unit assembly
+
+1. Load all checkpoint rows; sort by `page_start`
+2. Concatenate into `book_stream`
+3. Subdivide using TOC map (when added) or Docling `#` / `##` headers
+4. `logical_unit_id` = `{section_number}` or `{header_slug}`
+
+### Boundary sources
+
+- TOC map: TBD (`corpus/pdf/ado/index.txt` or equivalent)
+- Fallback: markdown headers in Docling output
+
+---
+
+## TNY
+
+### Physical split
+
+- Script: `knowledge/tests/tny_split_page.py`
+- Master PDF: `corpus/pdf/tny/tny_bounded.pdf`
+- Output: `corpus/pdf/tny/batches/`
+- Batch size: 5 pages
+- Filename: `batch-{NN}_chunk-{PPP}-{PPP}.pdf`
+- Example: `batch-02_chunk-006-010.pdf`
+
+### Filename regex
+
+```regex
+^batch-(?P<batch_index>\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$
+```
+
+### Parsed metadata example
+
+```json
+{
+  "book_id": "tny",
+  "batch_index": 2,
+  "page_start": 6,
+  "page_end": 10,
+  "section_id": null,
+  "subsection_id": null
+}
+```
+
+### Logical unit assembly
+
+Same as ADO (loose tier):
+
+1. Stitch all batches in page order
+2. Detect section boundaries via TOC or headers
+3. `logical_unit_id` = `sec_{N}` or `h2_{slug}`
+
+### Boundary sources
+
+- TOC: TBD
+- NER glossary (separate concern): `corpus/ner_glossary/TNY_*.txt`
+
+### Checkpoint DB
+
+`tny_ingestion_corpus.db` (see `knowledge/tests/ingestion_corpus_db.py`)
+
+---
+
+## OHO
+
+### Physical split
+
+- Script: `knowledge/tests/oho_split_page.py`
+- Master PDF: `corpus/pdf/oho/oho_bounded.pdf`
+- Section index: `corpus/pdf/oho/index.txt`
+- Output: `corpus/pdf/oho/sections/`
+- Batch size: 5 pages **within each section** (does not cross section boundaries)
+- Filename: `sec_{N}_batch-{NN}_chunk-{PPP}-{PPP}.pdf`
+- Example: `sec_3_batch-01_chunk-104-108.pdf`
+
+### Section index format (`index.txt`)
+
+```text
+section 1: 32-50
+section 2: 51-103
+section 3: 104-243
+...
+```
+
+### Filename regex
+
+```regex
+^sec_(?P<section_id>\d+)_batch-(?P<batch_index>\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$
+```
+
+### Parsed metadata example
+
+```json
+{
+  "book_id": "oho",
+  "section_id": 3,
+  "batch_index": 1,
+  "page_start": 104,
+  "page_end": 108,
+  "subsection_id": null
+}
+```
+
+### Logical unit assembly
+
+1. Group checkpoint rows by `section_id`
+2. Sort by `batch_index` (then `page_start`)
+3. Concatenate within group
+4. `logical_unit_id` = `sec_{section_id}`
+
+**Never** merge rows across different `section_id` values.
+
+### Boundary sources
+
+- Hard: `corpus/pdf/oho/index.txt` (section page ranges)
+- Soft: `##` / `###` headers within assembled section text
+- Index markdown (separate artifact): `corpus/indexs/oho_index_clean.md`
+
+### Checkpoint DB
+
+`oho_ingestion_corpus.db`
+
+---
+
+## MOR
+
+### Physical split
+
+- Script: `knowledge/tests/mor_split_page.py`
+- Master PDF: `corpus/pdf/mor/mor_bounded.pdf`
+- TOC: `corpus/pdf/mor/toc_structure.json`
+- Output: `corpus/pdf/mor/sections/`
+- Unit: one PDF per **subsection** (page count varies)
+- Filename: `sub_{subsection_id}_chunk-{PPP}-{PPP}.pdf`
+- Example: `sub_1.1_chunk-028-032.pdf`
+
+### Filename regex
+
+```regex
+^sub_(?P<subsection_id>\d+\.\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$
+```
+
+### Parsed metadata example
+
+```json
+{
+  "book_id": "mor",
+  "subsection_id": "1.1",
+  "section_id": 1,
+  "page_start": 28,
+  "page_end": 32,
+  "batch_index": null
+}
+```
+
+Derive `section_id` as integer part of `subsection_id` (`1.1` → section `1`).
+
+### Logical unit assembly
+
+**No cross-row assembly.** Each `processed_chunks` row maps 1:1 to a logical unit.
+
+- `logical_unit_id` = `sub_{subsection_id}`
+- `parent_title` from `toc_structure.json` → matching subsection `title`
+
+### TOC lookup
+
+```json
+{
+  "sections": [
+    {
+      "section_number": 1,
+      "section_title": "SECTION I: GENERAL",
+      "subsections": [
+        {
+          "title": "1.1 History Taking for Patients with Rheumatic Complaints",
+          "page_range": "28-32"
+        }
+      ]
+    }
+  ]
+}
+```
+
+Match `subsection_id` prefix to subsection `title` (e.g. `1.1` → title starting with `1.1`).
+
+### Checkpoint DB
+
+`mor_ingestion_corpus.db`
+
+---
+
+## Metadata parser interface
+
+Future module: `knowledge/implementation/ingestion/metadata_parser.py`
+
+```python
+@dataclass
+class SourceMetadata:
+    book_id: str
+    page_start: int
+    page_end: int
+    section_id: int | None
+    subsection_id: str | None
+    batch_index: int | None
+    source_filename: str
+
+def parse_source_path(source_path: str) -> SourceMetadata: ...
+
+def logical_unit_key(meta: SourceMetadata) -> str:
+    """Stable assembly key for grouping checkpoint rows."""
+    if meta.book_id == "mor":
+        return f"sub_{meta.subsection_id}"
+    if meta.book_id == "oho":
+        return f"sec_{meta.section_id}"
+    # ado, tny — assigned after book_stream section detection
+    raise NotImplementedError("loose tier uses post-assembly logical_unit_id")
+```
+
+## Rollout validation matrix
+
+| Book | Validate assembly | Validate chunk count | Validate citations |
+|------|-------------------|----------------------|--------------------|
+| MOR | 1 row → 1 unit | ~1 chunk per short subsection | page_range matches TOC |
+| OHO | N batches → 1 unit per section | multiple chunks per long section | section_id matches index.txt |
+| TNY/ADO | all rows → section units | headers align with visual TOC | page ranges contiguous |
+
+## References
+
+- `ingestion_pipeline_spec.md`
+- `semantic_chunking_spec.md`
+- `schema.md`

workspace/sprint_1_2/CODEBASE/knowledge/spec/ingestion/ingestion_pipeline_spec.md

@@ -0,0 +1,155 @@
+# Knowledge Ingestion Pipeline Specification
+
+## Purpose
+
+Defines the end-to-end pipeline that turns clinical textbook PDFs into retrieval-ready semantic chunks with traceable citations. This spec covers Stages 0–4; Stage 3 (semantic chunking) is detailed in `semantic_chunking_spec.md`.
+
+## Owner
+
+Knowledge Engineering Team
+
+## Scope
+
+| In scope | Out of scope |
+|----------|--------------|
+| PDF physical splitting for Docling | Qdrant collection tuning |
+| Docling markdown extraction checkpoint (SQLite) | LLM grounding / RAG answer generation |
+| Book-aware logical-unit assembly | ladybugDB ontology ingestion |
+| Header + semantic chunking | Production embedding server deployment |
+| Chunk metadata schema | Frontend citation UI |
+
+## Pipeline Stages
+
+```mermaid
+flowchart LR
+  S0[Stage 0: PDF split] --> S1[Stage 1: Docling extract]
+  S1 --> S2[(Stage 2: SQLite checkpoint)]
+  S2 --> S3[Stage 3: Semantic chunk]
+  S3 --> S4[(Stage 4: Embed + Qdrant)]
+```
+
+### Stage 0 — Physical PDF split
+
+Splits master bounded PDFs into Docling-sized slices. **Physical splits are for extraction parallelism only; they are not RAG chunk boundaries.**
+
+| Book | Constraint tier | Split rule | Output dir | Reference script |
+|------|-----------------|------------|------------|------------------|
+| **ADO** | Loose | Fixed 5-page batches, no section boundary | `corpus/pdf/ado/batches/` | (same pattern as TNY) |
+| **TNY** | Loose | Fixed 5-page batches, no section boundary | `corpus/pdf/tny/batches/` | `knowledge/tests/tny_split_page.py` |
+| **OHO** | Section | 5-page batches **within** each section | `corpus/pdf/oho/sections/` | `knowledge/tests/oho_split_page.py` |
+| **MOR** | Subsection | One PDF per TOC subsection (variable page count) | `corpus/pdf/mor/sections/` | `knowledge/tests/mor_split_page.py` |
+
+Section/subsection page maps:
+
+- OHO: `corpus/pdf/oho/index.txt`
+- MOR: `corpus/pdf/mor/toc_structure.json`
+
+### Stage 1 — Docling extraction
+
+Convert each physical PDF slice to markdown.
+
+- **Engine:** Docling `DocumentConverter` (pre-loaded singleton for batch runs)
+- **Output:** Markdown string per source file
+- **Reference:** `knowledge/tests/ingestion_corpus_db.py`
+
+Constraints:
+
+- One SQLite row per physical PDF file (never merge at this stage)
+- Preserve full `source_path` for downstream metadata parsing
+- Idempotent: skip files already present in `processed_chunks` unless `force_reextract=true`
+
+### Stage 2 — Extraction checkpoint (SQLite)
+
+Raw markdown checkpoint. **Do not semantic-chunk or embed directly from this table.**
+
+See `schema.md` → `processed_chunks`.
+
+Per-book databases (convention):
+
+| Book | DB file |
+|------|---------|
+| ADO | `ado_ingestion_corpus.db` |
+| TNY | `tny_ingestion_corpus.db` |
+| OHO | `oho_ingestion_corpus.db` |
+| MOR | `mor_ingestion_corpus.db` |
+
+A unified `{book}_ingestion_corpus.db` per textbook keeps checkpoint isolation and allows independent re-chunking.
+
+### Stage 3 — Semantic chunking
+
+Transform checkpoint rows into retrieval-sized chunks using book-aware assembly + hybrid splitting.
+
+**Detailed rules:** `semantic_chunking_spec.md`  
+**Book profiles:** `corpus_profiles.md`
+
+Input: `processed_chunks` rows for one book  
+Output: `semantic_chunks` rows (same DB or sibling DB)
+
+Properties:
+
+- Versioned (`chunker_version`) — re-chunk without re-extracting
+- Idempotent — skip logical units already chunked at current version
+- Traceable — every chunk links back to one or more `processed_chunks.id`
+
+### Stage 4 — Embedding and vector store
+
+Embed `semantic_chunks.content` and upsert to Qdrant (768-d, EmbeddingGemma per `knowledge_spec.md`).
+
+Payload fields (minimum):
+
+```json
+{
+  "chunk_id": "uuid",
+  "book_id": "mor",
+  "logical_unit_id": "1.1",
+  "parent_title": "1.1 History Taking for Patients with Rheumatic Complaints",
+  "page_start": 28,
+  "page_end": 32,
+  "chunk_index": 0,
+  "chunker_version": "header+semantic_v1",
+  "source_extraction_ids": [12]
+}
+```
+
+Update `semantic_chunks.embedding_status` and `qdrant_point_id` on success.
+
+## Planned Implementation Layout
+
+```
+knowledge/
+├─ spec/ingestion/           # This spec package
+├─ implementation/ingestion/ # Stage 1–4 modules (future)
+│   ├─ extract.py            # Docling → processed_chunks
+│   ├─ assemble.py           # Book-aware logical-unit assembly
+│   ├─ chunk.py              # Header + semantic chunking
+│   ├─ embed.py              # EmbeddingGemma → Qdrant
+│   └─ metadata_parser.py    # source_path → structured metadata
+└─ tests/
+    ├─ ingestion_corpus_db.py    # Stage 1–2 (existing)
+    ├─ *_split_page.py             # Stage 0 (existing)
+    └─ markdown_chunking_oho.py    # Prototype header split (superseded by chunk.py)
+```
+
+## Rollout Order
+
+1. **MOR** — subsection rows ≈ logical units; validate chunk → embed → retrieve loop
+2. **OHO** — section-scoped batch assembly from filenames
+3. **TNY / ADO** — full-book stream assembly + header/TOC boundary detection
+
+## Breaking-change Policy
+
+| Change | Version bump |
+|--------|--------------|
+| New book profile | MINOR |
+| `chunker_version` algorithm change | New version string; re-chunk required |
+| `semantic_chunks` schema additive column | MINOR |
+| Embedding model dimension change | MAJOR (per `knowledge_spec.md`) |
+| Removal of citation metadata field | MAJOR |
+
+## References
+
+- `semantic_chunking_spec.md` — Stage 3 algorithm
+- `corpus_profiles.md` — per-book assembly and filename parsing
+- `schema.md` — SQLite table definitions
+- `../knowledge_spec.md` — RAG stack (Qdrant, EmbeddingGemma)
+- `knowledge/tests/ingestion_corpus_db.py` — current Stage 1–2 prototype

workspace/sprint_1_2/CODEBASE/knowledge/spec/ingestion/schema.md

@@ -0,0 +1,190 @@
+# Ingestion SQLite Schema
+
+## Purpose
+
+Defines checkpoint and semantic-chunk tables for the knowledge ingestion pipeline. One database file per textbook.
+
+## Database files
+
+| Book | Filename |
+|------|----------|
+| ADO | `ado_ingestion_corpus.db` |
+| TNY | `tny_ingestion_corpus.db` |
+| OHO | `oho_ingestion_corpus.db` |
+| MOR | `mor_ingestion_corpus.db` |
+
+Location: `knowledge/corpus/db/` (recommended) or co-located with ingestion scripts during development.
+
+---
+
+## Table: `processed_chunks` (Stage 2 — extraction checkpoint)
+
+Raw Docling markdown per physical PDF slice. **Read-only input for semantic chunking.**
+
+```sql
+CREATE TABLE IF NOT EXISTS processed_chunks (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    source_path     TEXT NOT NULL UNIQUE,
+    markdown_content TEXT NOT NULL,
+    execution_time  REAL,
+    processed_at    TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE INDEX IF NOT EXISTS idx_processed_chunks_source_path
+    ON processed_chunks (source_path);
+```
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `id` | INTEGER | Surrogate key; referenced by `semantic_chunks.source_extraction_ids` |
+| `source_path` | TEXT | Absolute or repo-relative path to source PDF |
+| `markdown_content` | TEXT | Full Docling markdown for that PDF slice |
+| `execution_time` | REAL | Docling wall time (seconds) |
+| `processed_at` | TIMESTAMP | Insert timestamp |
+
+**Migration from current prototype:** `knowledge/tests/ingestion_corpus_db.py` already creates this table; add `UNIQUE` on `source_path` and index when promoting to implementation.
+
+---
+
+## Table: `semantic_chunks` (Stage 3 — retrieval chunks)
+
+Hybrid header + semantic chunks ready for embedding.
+
+```sql
+CREATE TABLE IF NOT EXISTS semantic_chunks (
+    id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+    chunk_uuid              TEXT NOT NULL UNIQUE,
+    logical_unit_id         TEXT NOT NULL,
+    chunk_index             INTEGER NOT NULL,
+    content                 TEXT NOT NULL,
+    token_count             INTEGER NOT NULL,
+    book_id                 TEXT NOT NULL,
+    section_id              INTEGER,
+    subsection_id           TEXT,
+    parent_title            TEXT,
+    page_start              INTEGER,
+    page_end                INTEGER,
+    source_extraction_ids   TEXT NOT NULL,  -- JSON array of processed_chunks.id
+    chunker_version         TEXT NOT NULL,
+    embedding_status        TEXT NOT NULL DEFAULT 'pending',
+    qdrant_point_id         TEXT,
+    content_hash            TEXT NOT NULL,
+    created_at              TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+
+    UNIQUE (logical_unit_id, chunk_index, chunker_version, book_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_semantic_chunks_book_version
+    ON semantic_chunks (book_id, chunker_version);
+
+CREATE INDEX IF NOT EXISTS idx_semantic_chunks_embedding_status
+    ON semantic_chunks (embedding_status);
+
+CREATE INDEX IF NOT EXISTS idx_semantic_chunks_logical_unit
+    ON semantic_chunks (logical_unit_id, chunker_version);
+```
+
+| Column | Type | Description |
+|--------|------|-------------|
+| `chunk_uuid` | TEXT | UUID v4 for Qdrant payload / API citations |
+| `logical_unit_id` | TEXT | Assembly key, e.g. `sec_3`, `sub_1.1`, `tny_sec_12` |
+| `chunk_index` | INTEGER | Order within logical unit (0-based) |
+| `content` | TEXT | Final chunk markdown |
+| `token_count` | INTEGER | Tokens per embedding model tokenizer |
+| `book_id` | TEXT | `ado`, `tny`, `oho`, `mor` |
+| `section_id` | INTEGER | Nullable; from filename or TOC |
+| `subsection_id` | TEXT | Nullable; e.g. `1.1` (MOR) |
+| `parent_title` | TEXT | Human-readable section/subsection title |
+| `page_start`, `page_end` | INTEGER | Inclusive citation page range |
+| `source_extraction_ids` | TEXT | JSON `[12, 13, 14]` — provenance |
+| `chunker_version` | TEXT | e.g. `header+semantic_v1` |
+| `embedding_status` | TEXT | `pending` \| `done` \| `failed` |
+| `qdrant_point_id` | TEXT | Set after Stage 4 upsert |
+| `content_hash` | TEXT | SHA-256 of `content` for dedup |
+
+### `embedding_status` values
+
+| Value | Meaning |
+|-------|---------|
+| `pending` | Chunk written; not yet embedded |
+| `done` | Embedded and upserted to Qdrant |
+| `failed` | Embed/upsert error; retry eligible |
+
+---
+
+## Table: `chunking_runs` (optional — audit log)
+
+Track chunk pipeline executions for idempotency and debugging.
+
+```sql
+CREATE TABLE IF NOT EXISTS chunking_runs (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    book_id         TEXT NOT NULL,
+    chunker_version TEXT NOT NULL,
+    started_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    finished_at     TIMESTAMP,
+    status          TEXT NOT NULL DEFAULT 'running',
+    units_processed INTEGER DEFAULT 0,
+    chunks_written  INTEGER DEFAULT 0,
+    error_message   TEXT,
+
+    UNIQUE (book_id, chunker_version, started_at)
+);
+```
+
+---
+
+## PostgreSQL target (Supabase)
+
+Stage 4 upserts into the **`knowledge`** schema on Supabase (pgvector HNSW). See:
+
+- `../pg_semantic_vector_db/er_diagram.md` — entity model
+- `../pg_semantic_vector_db/supabase_schema.md` — migration reference
+- `../../supabase/migrations/` — apply with `supabase db push`
+
+| SQLite column | Supabase table.column |
+|---------------|----------------------|
+| `chunk_uuid` | `knowledge.semantic_chunk.chunk_id` |
+| `embedding_status` | `knowledge.semantic_chunk_embedding.embedding_status` |
+| `qdrant_point_id` | **dropped** — use `chunk_id` |
+
+---
+
+## Qdrant payload mapping (legacy reference)
+
+| Qdrant payload field | Source column |
+|----------------------|---------------|
+| `chunk_id` | `chunk_uuid` |
+| `book_id` | `book_id` |
+| `logical_unit_id` | `logical_unit_id` |
+| `parent_title` | `parent_title` |
+| `page_start` | `page_start` |
+| `page_end` | `page_end` |
+| `chunk_index` | `chunk_index` |
+| `chunker_version` | `chunker_version` |
+| `section_id` | `section_id` |
+| `subsection_id` | `subsection_id` |
+
+Vector: 768 dimensions (EmbeddingGemma).
+
+Collection naming convention: `guidelines_{book_id}_{chunker_version}` (e.g. `guidelines_mor_header+semantic_v1`).
+
+---
+
+## Idempotency rules
+
+| Operation | Skip condition |
+|-----------|----------------|
+| Extract (Stage 1–2) | `source_path` already in `processed_chunks` |
+| Chunk (Stage 3) | `(logical_unit_id, chunk_index, chunker_version, book_id)` exists |
+| Embed (Stage 4) | `embedding_status = 'done'` |
+
+Force re-run: pass `--force` flag (implementation) to delete and regenerate rows for scoped book + version.
+
+---
+
+## References
+
+- `ingestion_pipeline_spec.md`
+- `semantic_chunking_spec.md`
+- `corpus_profiles.md`

workspace/sprint_1_2/CODEBASE/knowledge/spec/ingestion/semantic_chunking_spec.md

@@ -0,0 +1,198 @@
+# Semantic Chunking Specification
+
+## Purpose
+
+Defines how raw Docling markdown (stored in `processed_chunks`) is transformed into retrieval-ready semantic chunks. Applies a **hybrid strategy**: book-aware assembly → structural pre-split → embedding-based semantic split.
+
+## Owner
+
+Knowledge Engineering Team
+
+## Design Principles
+
+1. **Physical PDF splits ≠ RAG chunks.** Extraction batches exist for Docling memory and parallelism.
+2. **Structure before semantics.** Markdown headers and TOC boundaries are hard guardrails; semantic splitting refines oversized pieces only.
+3. **Never split inside protected blocks.** Tables, code fences, and figure captions stay intact.
+4. **Every chunk is citable.** Page range, book, logical unit, and source extraction IDs are mandatory metadata.
+5. **Re-chunkable.** `chunker_version` allows algorithm updates without re-running Docling.
+
+## Chunking Pipeline
+
+```mermaid
+flowchart TD
+  A[Read processed_chunks for book] --> B[Parse source_path metadata]
+  B --> C[Assemble logical units]
+  C --> D[Header pre-split]
+  D --> E{token_count > max_tokens?}
+  E -->|no| F[Emit chunk as-is]
+  E -->|yes| G[Semantic split]
+  G --> H[Enforce min_tokens / merge tiny pieces]
+  F --> I[Write semantic_chunks]
+  H --> I
+```
+
+### Step 1 — Parse `source_path`
+
+Extract structured metadata from each checkpoint row's filename. Regex patterns are defined in `corpus_profiles.md`.
+
+Minimum parsed fields:
+
+| Field | Description |
+|-------|-------------|
+| `book_id` | `ado`, `tny`, `oho`, `mor` |
+| `page_start`, `page_end` | Inclusive page numbers from filename |
+| `section_id` | Section number (OHO) or null |
+| `subsection_id` | Subsection id e.g. `1.1` (MOR) or null |
+| `batch_index` | Intra-unit batch sequence or null |
+
+### Step 2 — Assemble logical units
+
+Group checkpoint rows into **logical units** before any splitting. Assembly rules differ by constraint tier (see `corpus_profiles.md`).
+
+| Tier | Books | Assembly rule |
+|------|-------|---------------|
+| Loose | ADO, TNY | Stitch **all** batches in page order → one stream per book; then subdivide by headers/TOC |
+| Section | OHO | Group by `section_id`; stitch batches within section in page order |
+| Subsection | MOR | **No cross-row assembly** — each row is one logical unit |
+
+Concatenation separator between stitched batches:
+
+```text
+\n\n<!-- batch-boundary -->\n\n
+```
+
+The HTML comment marker is stripped from final chunk content but logged in assembly metadata for debugging mid-batch topic splits.
+
+### Step 3 — Header pre-split
+
+Split assembled text on markdown headers before semantic chunking.
+
+Default header levels:
+
+| Level | Metadata key | Use |
+|-------|--------------|-----|
+| `#` | `h1` | Part / major division (ADO, TNY when detected) |
+| `##` | `h2` | Section |
+| `###` | `h3` | Subsection |
+| `<!-- image --> ` | `<!-- image -->`  | Image|
+
+Implementation reference: LangChain `MarkdownHeaderTextSplitter` (prototype in `knowledge/tests/markdown_chunking_oho.py`).
+
+Rules:
+
+- Propagate header metadata to all descendant chunks (`parent_title` = nearest `##` or `###`)
+- If a header section is empty after strip, discard it
+- Do not split inside fenced code blocks (```) or HTML tables
+
+### Step 4 — Semantic split (oversized pieces only)
+
+Apply embedding-based breakpoint detection **only** when `token_count(content) > max_tokens`.
+
+| Parameter | Default | Notes |
+|-----------|---------|-------|
+| `max_tokens` | 800 | Target upper bound per chunk |
+| `min_tokens` | 120 | Merge or carry forward pieces below this |
+| `overlap_tokens` | 64 | Optional overlap at semantic boundaries |
+| `breakpoint_threshold_type` | `percentile` | Alternative: `standard_deviation`, `interquartile` |
+| `breakpoint_threshold_amount` | 95 | Percentile for similarity drop detection |
+| `embedding_model` | EmbeddingGemma | Must match Stage 4 embed model |
+
+Recommended library: LangChain Experimental `SemanticChunker` or LlamaIndex `SemanticSplitterNodeParser`.
+
+Sentence/paragraph granularity:
+
+- Split input into sentences (or paragraphs for list-heavy clinical text)
+- Embed consecutive windows; detect largest cosine-similarity drops
+- Cut at breakpoints; merge segments below `min_tokens` with neighbors
+
+**MOR shortcut:** If logical unit token count ≤ `max_tokens`, skip semantic split entirely (pass-through after optional header split).
+
+### Step 5 — Post-processing
+
+1. **Merge tiny chunks:** If `token_count < min_tokens`, merge with previous chunk in same logical unit (unless previous would exceed `max_tokens * 1.25`)
+2. **Assign `chunk_index`:** Zero-based, ordered within logical unit
+3. **Compute page range:** For multi-batch units, use min(`page_start`) and max(`page_end`) across constituent extraction rows; for semantic sub-chunks within a unit, inherit parent page range (fine-grained page mapping is a future enhancement)
+4. **Deduplicate:** Hash `(book_id, logical_unit_id, chunk_index, chunker_version, content)` — skip exact duplicates on re-run
+
+## Chunker Versions
+
+| Version | Description |
+|---------|-------------|
+| `header_v1` | Header pre-split only; no semantic split |
+| `header+semantic_v1` | Header pre-split + semantic split on oversized pieces (default) |
+
+Bump version string when algorithm parameters or assembly rules change materially.
+
+## Book-specific behavior summary
+
+### ADO / TNY (loose tier)
+
+Highest assembly effort:
+
+1. Load all `processed_chunks` ordered by `page_start`
+2. Concatenate into `book_stream`
+3. Subdivide `book_stream`:
+   - **Preferred:** TOC page→section map (when available)
+   - **Fallback:** Split on `#` / `##` headers from Docling output
+4. Header pre-split within each section
+5. Semantic split oversized sections
+
+Risk: topic spans batch boundary with no header at boundary → semantic chunker must see full assembled section text, not individual batches.
+
+### OHO (section tier)
+
+1. Group rows by `section_id` from filename (`sec_N_...`)
+2. Sort by `batch_index` / `page_start`; concatenate
+3. Header pre-split within section
+4. Semantic split if section text > `max_tokens`
+
+Section boundaries from `corpus/pdf/oho/index.txt` are authoritative — never merge across sections.
+
+### MOR (subsection tier)
+
+1. Each row = one logical unit (`subsection_id` from `sub_X.X_...`)
+2. Optional header pre-split on `##` / `###` within subsection
+3. Semantic split only if subsection > `max_tokens`
+4. Many subsections (≤5 pages) may produce a **single chunk**
+
+Subsection titles from `corpus/pdf/mor/toc_structure.json` populate `parent_title`.
+
+## Protected content rules
+
+Do not split inside:
+
+- Markdown tables (`| ... |` blocks)
+- Fenced code blocks
+- Numbered/bulleted lists shorter than `min_tokens` (keep list intact)
+- Figure/table captions immediately following a figure reference line
+
+If semantic split would break a table, prefer keeping the entire table in one chunk even if it slightly exceeds `max_tokens` (hard cap: `max_tokens * 1.5`).
+
+## Output contract
+
+Each row in `semantic_chunks` must satisfy:
+
+- `content` is non-empty UTF-8 markdown
+- `token_count` is computed with the same tokenizer as the embedding model
+- `source_extraction_ids` is a JSON array of one or more `processed_chunks.id`
+- `logical_unit_id` is stable across re-runs (see `corpus_profiles.md`)
+- `chunker_version` matches the active pipeline version
+
+## Evaluation criteria (acceptance)
+
+Before promoting a new `chunker_version`:
+
+| Check | Target |
+|-------|--------|
+| Sample retrieval on 20 clinical queries | ≥ baseline vs header-only |
+| Chunks with `token_count < min_tokens` | < 5% of total |
+| Chunks with `token_count > max_tokens * 1.5` | < 2% of total |
+| Chunks missing `parent_title` (MOR/OHO) | 0% |
+| Citation traceability (manual spot check) | 100% link to source pages |
+
+## References
+
+- `ingestion_pipeline_spec.md` — full pipeline context
+- `corpus_profiles.md` — filename patterns and assembly keys
+- `schema.md` — `semantic_chunks` table
+- `knowledge/tests/markdown_chunking_oho.py` — header split prototype

workspace/sprint_1_2/CODEBASE/knowledge/spec/interface_contract.md

@@ -28,7 +28,7 @@ Knowledge Engineering Team
 - ladybugDB schema details (predicate names, ontology version)
 - embedding model specifics (model ID, tensor shapes)
 - LLM prompt templates and decoding parameters
-- knowledge curation pipeline details (source ingestion, validation)
+- ingestion SQLite checkpoint paths and chunker version strings (see `spec/ingestion/` for pipeline design; operational paths are internal)
 
 ## Breaking-change Policy
 - Knowledge schema versioning via semantic versioning (MAJOR.MINOR.PATCH)

workspace/sprint_1_2/CODEBASE/knowledge/spec/knowledge_spec.md

@@ -18,12 +18,33 @@ Qdrant vector database instances, ladybugDB graph database instances, embedding
 - Retrieval pipeline: hybrid search (vector + BM25) → graph expansion → reranking
 - Grounding module: verifies LLM outputs against source guidelines with citation extraction
 - Arbitration engine: resolves conflicting evidence using belief propagation
-- Continuous integration: automated guideline ingestion from trusted sources (NIH, CDC, radiology societies)
+- Textbook ingestion pipeline: PDF split → Docling checkpoint (SQLite) → book-aware semantic chunking → pgvector upsert (see `spec/ingestion/`, `spec/pg_semantic_vector_db/`)
 - Versioned knowledge bases with temporal validity tracking
 - Monitoring: retrieval relevance, grounding accuracy, latency SLOs
 
+## Ingestion Specifications
+See `spec/ingestion/` for the textbook RAG ingestion pipeline:
+
+| Document | Scope |
+|----------|-------|
+| `ingestion_pipeline_spec.md` | Stages 0–4 end-to-end |
+| `semantic_chunking_spec.md` | Hybrid header + semantic chunking algorithm |
+| `corpus_profiles.md` | ADO, TNY, OHO, MOR assembly and filename rules |
+| `schema.md` | SQLite `processed_chunks` and `semantic_chunks` tables |
+
+## Vector Store Specifications
+
+See `spec/pg_semantic_vector_db/` for the PostgreSQL + pgvector retrieval schema:
+
+| Document | Scope |
+|----------|-------|
+| `er_diagram.md` | Entities, relationships, and SQLite→PG mapping |
+| `supabase_schema.md` | Applied Supabase migrations, RPC, security model |
+
+Migrations: `knowledge/supabase/migrations/`
+
 ## Interface Contract
-See `bento/knowledge/spec/interface-contract.md`.
+See `spec/interface_contract.md`.
 
 ## Consumers
 - frontend:guideline-spec (for displaying grounded explanations in UI)

workspace/sprint_1_2/CODEBASE/knowledge/spec/pg_semantic_vector_db/er_diagram.md

@@ -0,0 +1,509 @@
+# pg-Semantic-Vector-DB — Entity-Relationship Design
+
+## Purpose
+
+Define the **PostgreSQL + pgvector** data model for retrieval-ready semantic chunks from clinical textbooks (ADO, TNY, OHO, MOR). This is the **authoritative RAG store** queried at runtime (HNSW similarity search + SQL metadata filters).
+
+## Scope
+
+| In scope | Out of scope (separate stores) |
+|----------|--------------------------------|
+| Textbook semantic chunks and their EmbeddingGemma vectors | SQLite ingestion checkpoints (`processed_chunks`) |
+| Citation metadata for NFR-18 grounding | BERT drift/referee embeddings (separate `drift_embeddings` table per architecture) |
+| Corpus edition and chunker versioning | ladybugDB ontology graph |
+| Provenance links back to extraction checkpoints | Session/case embeddings (clinical data domain) |
+| HNSW index lifecycle per corpus + model | Qdrant (Phase 2 hot cache only) |
+
+## Storage boundary
+
+```mermaid
+flowchart LR
+  subgraph sqlite [SQLite — ingestion staging]
+    PC[processed_chunks]
+    SC_STG[semantic_chunks staging]
+  end
+  subgraph pg [PostgreSQL + pgvector — retrieval]
+    CE[corpus_edition]
+    LU[logical_unit]
+    SC[semantic_chunk]
+    SCE[semantic_chunk_embedding]
+  end
+  PC -->|Stage 3 chunk| SC_STG
+  SC_STG -->|Stage 4 upsert| SC
+  SC --> SCE
+```
+
+SQLite remains **per-book, per-pipeline** checkpoint storage. PostgreSQL holds **deduplicated, queryable, versioned** chunks with vectors.
+
+---
+
+## ER diagram
+
+```mermaid
+erDiagram
+    corpus_source ||--o{ corpus_edition : "has editions"
+    corpus_edition ||--o{ structure_node : "has TOC tree"
+    structure_node ||--o{ structure_node : "parent of"
+    corpus_edition ||--o{ logical_unit : "contains"
+    structure_node |o--o| logical_unit : "may map to"
+    logical_unit ||--|{ semantic_chunk : "split into"
+    chunker_profile ||--o{ semantic_chunk : "produced by"
+    semantic_chunk ||--o{ chunk_provenance : "traced from"
+    semantic_chunk ||--o{ semantic_chunk_embedding : "embedded as"
+    embedding_model ||--o{ semantic_chunk_embedding : "uses"
+    corpus_edition ||--o{ ingestion_run : "processed in"
+    chunker_profile ||--o{ ingestion_run : "uses"
+    corpus_edition ||--o{ vector_index_snapshot : "indexed under"
+    embedding_model ||--o{ vector_index_snapshot : "indexed with"
+
+    corpus_source {
+        uuid corpus_source_id PK
+        text book_id UK "ado | tny | oho | mor"
+        text display_name
+        text constraint_tier "loose | section | subsection"
+        text locale "vi-VN"
+        timestamptz created_at
+    }
+
+    corpus_edition {
+        uuid edition_id PK
+        uuid corpus_source_id FK
+        text edition_label "e.g. 2024-print, v1"
+        text source_pdf_sha256
+        text status "draft | active | superseded | archived"
+        date effective_from
+        date effective_to
+        timestamptz created_at
+    }
+
+    structure_node {
+        uuid structure_node_id PK
+        uuid edition_id FK
+        uuid parent_structure_node_id FK "nullable — root nodes"
+        text node_type "part | section | subsection"
+        text node_key "e.g. sec_3, 1.1"
+        text title
+        int page_start
+        int page_end
+        int sort_order
+    }
+
+    logical_unit {
+        uuid logical_unit_id PK
+        uuid edition_id FK
+        uuid structure_node_id FK "nullable"
+        text unit_key "stable key e.g. sec_3, sub_1.1"
+        text parent_title
+        int page_start
+        int page_end
+        timestamptz assembled_at
+    }
+
+    chunker_profile {
+        text chunker_version PK "e.g. header+semantic_v1"
+        text description
+        int max_tokens
+        int min_tokens
+        int overlap_tokens
+        jsonb params_json
+        timestamptz created_at
+    }
+
+    semantic_chunk {
+        uuid chunk_id PK "API citation id"
+        uuid logical_unit_id FK
+        uuid edition_id FK
+        text chunker_version FK
+        int chunk_index "0-based within logical unit"
+        text content "markdown"
+        int token_count
+        text content_hash "SHA-256"
+        int section_id "nullable"
+        text subsection_id "nullable"
+        text parent_title
+        int page_start
+        int page_end
+        boolean is_active "soft retire on re-chunk"
+        timestamptz created_at
+    }
+
+    chunk_provenance {
+        uuid provenance_id PK
+        uuid chunk_id FK
+        text checkpoint_db "e.g. mor_ingestion_corpus.db"
+        bigint processed_chunk_id "SQLite processed_chunks.id"
+        text source_path "original PDF slice path"
+    }
+
+    embedding_model {
+        uuid model_id PK
+        text model_name "EmbeddingGemma"
+        text model_version
+        int dimensions "768"
+        text purpose "rag"
+        boolean is_active
+        timestamptz registered_at
+    }
+
+    semantic_chunk_embedding {
+        uuid embedding_id PK
+        uuid chunk_id FK
+        uuid model_id FK
+        uuid edition_id FK
+        vector embedding "pgvector, 768-d"
+        text embedding_status "pending | indexed | failed"
+        timestamptz embedded_at
+    }
+
+    ingestion_run {
+        uuid run_id PK
+        uuid edition_id FK
+        text chunker_version FK
+        uuid model_id FK "nullable until Stage 4"
+        text stage "chunk | embed | full"
+        text status "running | succeeded | failed"
+        int units_processed
+        int chunks_written
+        int embeddings_written
+        text error_message
+        timestamptz started_at
+        timestamptz finished_at
+    }
+
+    vector_index_snapshot {
+        uuid index_snapshot_id PK
+        uuid edition_id FK
+        uuid model_id FK
+        text index_name "e.g. hnsw_semantic_chunk_embedding_rag"
+        jsonb hnsw_params "m, ef_construction"
+        boolean is_active
+        timestamptz built_at
+    }
+```
+
+---
+
+## Entity definitions
+
+### 1. `corpus_source`
+
+Registry of ingestible textbook corpora. One row per book family.
+
+| Attribute | Notes |
+|-----------|-------|
+| `book_id` | Stable slug: `ado`, `tny`, `oho`, `mor` |
+| `constraint_tier` | Mirrors `corpus_profiles.md`: drives assembly rules at ingest |
+| `locale` | Default `vi-VN` for pilot textbooks |
+| `full_title`, `short_title`, `subtitle` | Bibliographic titles for citations |
+| `primary_language` | ISO-style language code (`en`, `vi`) |
+| `subject_area` | RAG classification (e.g. Rheumatology) |
+| `bibliography_json` | Extensible work metadata (keywords, resource_type) |
+
+Bibliographic seed data: [`corpus/source_metadata.md`](../../corpus/source_metadata.md).
+
+**Cardinality:** one `corpus_source` → many `corpus_edition`.
+
+---
+
+### 2. `corpus_edition`
+
+Immutable snapshot of a source document generation. Re-chunking and re-embedding are scoped to an edition.
+
+| Attribute | Notes |
+|-----------|-------|
+| `source_pdf_sha256` | Detects PDF replacement without silent drift |
+| `status` | Only one `active` edition per `corpus_source` at a time (enforced by partial unique index) |
+| `effective_from` / `effective_to` | Temporal validity for citations and audit |
+| `publisher_name`, `publisher_place` | Publication imprint |
+| `published_year`, `published_date` | Publication date for citations |
+| `isbn_13`, `isbn_10` | ISBN identifiers |
+| `edition_number`, `source_uri` | Edition label and canonical source file |
+| `bibliography_json` | Extensible edition metadata (LCCN, series, etc.) |
+
+**Why separate from `corpus_source`:** Same book (TNY) may be re-ingested after PDF correction; old chunks stay addressable for audit while new edition becomes `active`.
+
+---
+
+### 2b. `corpus_contributor`
+
+Authors, editors, and other credited contributors for a work or specific edition.
+
+| Attribute | Notes |
+|-----------|-------|
+| `corpus_source_id` | Required FK to work |
+| `edition_id` | NULL = work-level contributor; set for edition-specific overrides |
+| `role` | `author`, `editor`, `translator`, `compiler`, `institution` |
+| `full_name`, `sort_order` | Citation ordering |
+
+Seed data: [`corpus/source_metadata.md`](../../corpus/source_metadata.md).
+
+---
+
+### 3. `structure_node`
+
+Optional document hierarchy (TOC). Strongly populated for OHO/MOR; ADO/TNY may be partially inferred from headers post-assembly.
+
+| Attribute | Notes |
+|-----------|-------|
+| `node_type` | `part`, `section`, `subsection` |
+| `node_key` | Matches filename/TOC keys: `sec_3`, `1.1` |
+| Self-FK | Tree; root nodes have `parent_structure_node_id IS NULL` |
+
+**Relationship:** `structure_node` **0..1 — 0..1** `logical_unit` (a logical unit may exist without a TOC node when boundaries come from header detection only).
+
+---
+
+### 4. `logical_unit`
+
+The **assembly boundary** before chunking — the unit described in `semantic_chunking_spec.md`.
+
+| Attribute | Notes |
+|-----------|-------|
+| `unit_key` | Stable across runs: `sec_{N}`, `sub_{X.Y}`, `h2_{slug}` |
+| `edition_id` | Same `unit_key` in two editions = different rows |
+
+**Cardinality:** one `logical_unit` → one or many `semantic_chunk` (after header/semantic split).
+
+**Assembly mapping by tier:**
+
+| Tier | Books | `logical_unit` origin |
+|------|-------|------------------------|
+| Loose | ADO, TNY | One row per detected section after full-book stitch |
+| Section | OHO | One row per `section_id` |
+| Subsection | MOR | One row per `subsection_id` (= one checkpoint row) |
+
+---
+
+### 5. `chunker_profile`
+
+Version registry for the chunking algorithm. Not a runtime entity for queries, but FK anchor for reproducibility.
+
+| Attribute | Notes |
+|-----------|-------|
+| `chunker_version` | PK string: `header+semantic_v1` |
+| `params_json` | Thresholds, breakpoint settings, protected-block rules hash |
+
+**Cardinality:** one profile → many `semantic_chunk` rows (across editions and logical units).
+
+---
+
+### 6. `semantic_chunk`
+
+Core **retrieval text unit**. This is what RAG returns (content + citation metadata).
+
+| Attribute | Notes |
+|-----------|-------|
+| `chunk_id` | UUID v4 — external citation id (replaces SQLite `chunk_uuid` / Qdrant point id) |
+| `chunk_index` | Order within parent `logical_unit` |
+| `is_active` | Set `false` when superseded by re-chunk; keeps audit trail |
+| Citation fields | `parent_title`, `page_start`, `page_end`, `section_id`, `subsection_id` |
+
+**Uniqueness:**
+
+```sql
+UNIQUE (logical_unit_id, chunk_index, chunker_version)
+-- scoped implicitly by edition via logical_unit.edition_id
+```
+
+**Cardinality:** one chunk → zero or one active embedding per model (see below).
+
+---
+
+### 7. `chunk_provenance`
+
+Many-to-one trace from a semantic chunk back to SQLite extraction rows. Supports multi-batch assembly (ADO/TNY/OHO).
+
+| Attribute | Notes |
+|-----------|-------|
+| `checkpoint_db` | Which `{book}_ingestion_corpus.db` |
+| `processed_chunk_id` | FK into SQLite, not PostgreSQL |
+| `source_path` | Denormalized for debugging without opening SQLite |
+
+**Cardinality:** one `semantic_chunk` → **1..N** `chunk_provenance` (N > 1 when batches were stitched).
+
+---
+
+### 8. `embedding_model`
+
+Registry of embedding models allowed to populate the RAG index.
+
+| Attribute | Notes |
+|-----------|-------|
+| `purpose` | `rag` only in this schema; drift models live elsewhere |
+| `dimensions` | Must match pgvector column width (768 for EmbeddingGemma) |
+
+PoC: single active row (`EmbeddingGemma`, 768-d).
+
+---
+
+### 9. `semantic_chunk_embedding`
+
+**The vector store.** Separated from `semantic_chunk` so text can be re-embedded under a new model without duplicating content rows.
+
+| Attribute | Notes |
+|-----------|-------|
+| `embedding` | `vector(768)` with HNSW index |
+| `embedding_status` | Pipeline state: `pending` → `indexed` |
+| `edition_id` | Denormalized for partition-friendly queries |
+
+**Uniqueness:**
+
+```sql
+UNIQUE (chunk_id, model_id)
+```
+
+**Cardinality:** one chunk → at most one embedding per model.
+
+**Query pattern (runtime RAG):**
+
+```sql
+SELECT sc.chunk_id, sc.content, sc.parent_title, sc.page_start, sc.page_end,
+       1 - (sce.embedding <=> :query_vec) AS similarity
+FROM semantic_chunk_embedding sce
+JOIN semantic_chunk sc ON sc.chunk_id = sce.chunk_id
+JOIN corpus_edition ce ON ce.edition_id = sc.edition_id
+JOIN corpus_source cs ON cs.corpus_source_id = ce.corpus_source_id
+WHERE ce.status = 'active'
+  AND sc.is_active = TRUE
+  AND sce.embedding_status = 'indexed'
+  AND cs.book_id = ANY(:book_ids)
+ORDER BY sce.embedding <=> :query_vec
+LIMIT :k;
+```
+
+---
+
+### 10. `ingestion_run`
+
+Audit log for chunk and embed pipeline executions (PostgreSQL counterpart to SQLite `chunking_runs` + embed stage).
+
+| Attribute | Notes |
+|-----------|-------|
+| `stage` | `chunk`, `embed`, or `full` |
+| Counters | `units_processed`, `chunks_written`, `embeddings_written` |
+
+**Cardinality:** one edition may have many runs (idempotent re-runs, version bumps).
+
+---
+
+### 11. `vector_index_snapshot`
+
+Tracks HNSW index builds for blue/green reindex when `chunker_version` or `embedding_model` changes.
+
+| Attribute | Notes |
+|-----------|-------|
+| `is_active` | Only one active snapshot per `(edition_id, model_id)` |
+| `hnsw_params` | e.g. `{"m": 16, "ef_construction": 64}` |
+
+At PoC scale (~15K vectors), a single HNSW index on `semantic_chunk_embedding.embedding` filtered by `edition_id` is sufficient; this entity documents index lifecycle for future corpus growth.
+
+---
+
+## Relationship summary
+
+| From | To | Cardinality | Semantics |
+|------|-----|-------------|-----------|
+| `corpus_source` | `corpus_edition` | 1:N | Book has versioned PDF editions |
+| `corpus_edition` | `structure_node` | 1:N | Edition has TOC tree |
+| `structure_node` | `structure_node` | 1:N | Parent/child hierarchy |
+| `corpus_edition` | `logical_unit` | 1:N | Edition contains assembly units |
+| `structure_node` | `logical_unit` | 0..1:0..1 | Optional TOC alignment |
+| `logical_unit` | `semantic_chunk` | 1:N | Unit splits into retrieval chunks |
+| `chunker_profile` | `semantic_chunk` | 1:N | Algorithm version provenance |
+| `semantic_chunk` | `chunk_provenance` | 1:N | Trace to SQLite extractions |
+| `semantic_chunk` | `semantic_chunk_embedding` | 1:N | One row per embedding model |
+| `embedding_model` | `semantic_chunk_embedding` | 1:N | Model produces many vectors |
+| `corpus_edition` | `ingestion_run` | 1:N | Pipeline execution history |
+| `corpus_edition` | `vector_index_snapshot` | 1:N | Index rebuild history |
+
+---
+
+## Design decisions
+
+### Why split `semantic_chunk` and `semantic_chunk_embedding`?
+
+- Re-embed with a new model without cloning text rows
+- Keep inactive/historical embeddings while marking chunks inactive
+- HNSW index targets only the embedding table (smaller hot set)
+
+### Why `corpus_edition` instead of embedding `book_id` only?
+
+- Supports PDF corrections and temporal citation ("per TNY 2024 edition, p. 42")
+- Clean supersession: deactivate old edition chunks without deleting audit data
+- Aligns with architecture "versioned knowledge bases with temporal validity tracking"
+
+### Why keep provenance as a junction table?
+
+- ADO/TNY/OHO chunks often span multiple `processed_chunks` rows
+- SQLite checkpoint DBs stay per-book; provenance stores the cross-reference explicitly
+
+### Active-edition invariant
+
+At query time, RAG MUST filter:
+
+```sql
+ce.status = 'active' AND sc.is_active = TRUE AND sce.embedding_status = 'indexed'
+```
+
+Only one active edition per `corpus_source` prevents mixed-corpus retrieval.
+
+---
+
+## Mapping from SQLite staging (`schema.md`)
+
+| SQLite `semantic_chunks` | PostgreSQL |
+|--------------------------|------------|
+| `chunk_uuid` | `semantic_chunk.chunk_id` |
+| `logical_unit_id` | `logical_unit.unit_key` (+ FK via edition) |
+| `chunk_index` | `semantic_chunk.chunk_index` |
+| `content` | `semantic_chunk.content` |
+| `token_count` | `semantic_chunk.token_count` |
+| `book_id` | `corpus_source.book_id` (via edition) |
+| `section_id`, `subsection_id` | same columns on `semantic_chunk` |
+| `parent_title`, `page_start`, `page_end` | same |
+| `source_extraction_ids` (JSON) | `chunk_provenance` rows (1 per id) |
+| `chunker_version` | FK → `chunker_profile` |
+| `embedding_status` | `semantic_chunk_embedding.embedding_status` |
+| `qdrant_point_id` | **dropped** — `chunk_id` is the retrieval key |
+| `content_hash` | `semantic_chunk.content_hash` |
+
+---
+
+## Indexes (planned)
+
+| Index | Table | Purpose |
+|-------|-------|---------|
+| HNSW | `semantic_chunk_embedding.embedding` | Cosine similarity search |
+| `(edition_id, is_active)` | `semantic_chunk` | Filter active chunks per edition |
+| `(corpus_source_id) WHERE status = 'active'` | `corpus_edition` | Partial unique — one active edition |
+| `(logical_unit_id, chunk_index, chunker_version)` | `semantic_chunk` | Idempotent upsert |
+| `(chunk_id, model_id)` | `semantic_chunk_embedding` | Unique embedding per model |
+| `(book_id)` | `corpus_source` | Filter by textbook at join time |
+
+---
+
+## PoC entity subset
+
+For first implementation (MOR → OHO → TNY/ADO), minimum viable tables:
+
+1. `corpus_source`
+2. `corpus_edition`
+3. `logical_unit`
+4. `chunker_profile`
+5. `semantic_chunk`
+6. `chunk_provenance`
+7. `embedding_model`
+8. `semantic_chunk_embedding`
+
+Defer until needed: `structure_node`, `vector_index_snapshot` (use DDL-only HNSW), full `ingestion_run` UI.
+
+---
+
+## References
+
+- `../ingestion/schema.md` — SQLite staging schema
+- `../ingestion/semantic_chunking_spec.md` — logical unit and chunk contract
+- `../ingestion/corpus_profiles.md` — per-book assembly tiers
+- `../knowledge_spec.md` — RAG stack (pgvector HNSW, EmbeddingGemma 768-d)
+- `supabase_schema.md` — Supabase migration reference (`knowledge/supabase/migrations/`)
+- `../../../Design_Material/SPRINT_1_2_ARCHITECTURE_SPEC.md` — pgvector over Qdrant at PoC scale

workspace/sprint_1_2/CODEBASE/knowledge/spec/pg_semantic_vector_db/supabase_schema.md

@@ -0,0 +1,98 @@
+# Supabase Schema Reference — pg-Semantic-Vector-DB
+
+Maps the ER design to applied Supabase migrations under `knowledge/supabase/migrations/`.
+
+## Migration order
+
+| File | Contents |
+|------|----------|
+| `20260705000000_semantic_vector_db_schema.sql` | `knowledge` schema, 11 tables, constraints, HNSW index |
+| `20260705000001_semantic_vector_db_seed.sql` | ADO/TNY/OHO/MOR sources, `header+semantic_v1`, EmbeddingGemma |
+| `20260705000002_semantic_vector_db_rls_and_rpc.sql` | RLS, grants, `match_semantic_chunks()` RPC |
+| `20260705000003_expose_knowledge_api_schema.sql` | PostgREST grants for `knowledge` schema |
+| `20260705000004_chunker_profile_v2.sql` | `header+semantic_v2` chunker profile |
+| `20260705000005_corpus_bibliography.sql` | Bibliography columns, `corpus_contributor`, citation view |
+| `20260705000006_corpus_bibliography_seed.sql` | Bibliographic seed from `corpus/source_metadata.md` |
+
+## Schema: `knowledge`
+
+| Table / view | PK | Notable constraints |
+|-------|-----|---------------------|
+| `corpus_source` | `corpus_source_id` | `UNIQUE (book_id)`; bibliography columns |
+| `corpus_edition` | `edition_id` | Partial unique: one `active` per source; publisher/ISBN |
+| `corpus_contributor` | `contributor_id` | Authors/editors per work or edition |
+| `v_corpus_citation` | — | View: active edition + contributor string |
+| `structure_node` | `structure_node_id` | `UNIQUE (edition_id, node_key)` |
+| `logical_unit` | `logical_unit_id` | `UNIQUE (edition_id, unit_key)` |
+| `chunker_profile` | `chunker_version` | — |
+| `semantic_chunk` | `chunk_id` | `UNIQUE (logical_unit_id, chunk_index, chunker_version)` |
+| `chunk_provenance` | `provenance_id` | `UNIQUE (chunk_id, checkpoint_db, processed_chunk_id)` |
+| `embedding_model` | `model_id` | `UNIQUE (model_name, model_version)` |
+| `semantic_chunk_embedding` | `embedding_id` | `UNIQUE (chunk_id, model_id)`; HNSW on `embedding` |
+| `ingestion_run` | `run_id` | — |
+| `vector_index_snapshot` | `index_snapshot_id` | Partial unique: one active per edition+model |
+
+## Vector column
+
+```sql
+embedding extensions.vector(768)   -- EmbeddingGemma, cosine HNSW
+```
+
+Extension: `CREATE EXTENSION vector WITH SCHEMA extensions` (Supabase default pattern).
+
+## RPC: `knowledge.match_semantic_chunks`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `query_embedding` | `extensions.vector(768)` | required | Query vector from EmbeddingGemma |
+| `match_count` | `integer` | `5` | Top-k limit |
+| `filter_book_ids` | `text[]` | `NULL` | Restrict to `ado`, `tny`, `oho`, `mor` |
+| `filter_edition_ids` | `uuid[]` | `NULL` | Restrict to specific editions |
+
+Returns: chunk content, citation fields, `book_id`, `similarity` (1 − cosine distance).
+
+## Security model
+
+| Role | Access |
+|------|--------|
+| `service_role` | Full CRUD on all `knowledge.*` tables; executes RPC |
+| `authenticated` | SELECT on active chunks + indexed embeddings; executes RPC |
+| `anon` | Denied (no policies) |
+
+Ingestion pipeline and RAG coordinator should use **`service_role`** from the backend only.
+
+## SQLite staging → Supabase upsert keys
+
+| SQLite (`semantic_chunks`) | Supabase upsert key |
+|----------------------------|---------------------|
+| `book_id` | Resolve → `corpus_edition` via active edition for `corpus_source.book_id` |
+| `logical_unit_id` | `logical_unit.unit_key` + `edition_id` |
+| `chunk_index`, `chunker_version` | `semantic_chunk` unique constraint |
+| `source_extraction_ids[]` | One `chunk_provenance` row per id |
+| `chunk_uuid` | `semantic_chunk.chunk_id` (preserve on upsert) |
+
+## Activate a new edition
+
+```sql
+BEGIN;
+UPDATE knowledge.corpus_edition
+SET status = 'superseded', effective_to = CURRENT_DATE
+WHERE corpus_source_id = (SELECT corpus_source_id FROM knowledge.corpus_source WHERE book_id = 'mor')
+  AND status = 'active';
+
+INSERT INTO knowledge.corpus_edition (corpus_source_id, edition_label, source_pdf_sha256, status, effective_from)
+VALUES (
+    (SELECT corpus_source_id FROM knowledge.corpus_source WHERE book_id = 'mor'),
+    'v1',
+    '<sha256>',
+    'active',
+    CURRENT_DATE
+);
+COMMIT;
+```
+
+## References
+
+- [`er_diagram.md`](er_diagram.md) — entity definitions and ER diagram
+- [`../ingestion/schema.md`](../ingestion/schema.md) — SQLite checkpoint schema
+- [`../../supabase/README.md`](../../supabase/README.md) — deploy instructions

workspace/sprint_1_2/CODEBASE/knowledge/tests/bounded_pdf.py

@@ -0,0 +1,48 @@
+import fitz  # PyMuPDF
+
+# Open the original dictionary PDF
+input_path = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/tny.pdf"
+output_path = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/tny_bounded.pdf"
+
+doc = fitz.open(input_path)
+
+# Loop through every single page in the document
+for page_num in range(len(doc)):
+    page = doc[page_num]
+    
+    # 1. Dynamically calculate center per page
+    page_width = page.rect.width
+    page_height = page.rect.height
+    center_x = page_width / 2
+
+    # Draw the blue vertical column divider down the middle gutter
+    page.draw_line(
+        fitz.Point(center_x, 0), 
+        fitz.Point(center_x, page_height), 
+        color=(0, 0, 1), 
+        width=3
+    )
+
+    # 2. Get text blocks for this specific page
+    blocks = page.get_text("blocks")
+
+    for b in blocks:
+        x0, y0, x1, y1 = b[0], b[1], b[2], b[3]
+        
+        # OPTIONAL FILTER: Ignore running headers at the very top or footers at the bottom
+        # Adjust these numbers (e.g., 30 and page_height - 30) based on your document margins
+        if y0 < 40 or y1 > (page_height - 40):
+            continue
+            
+        # Draw a green bounding box around the whole paragraph block
+        page.draw_rect(
+            fitz.Rect(x0, y0, x1, y1), 
+            color=(0, 0.6, 0), 
+            width=1.5
+        )
+
+# Save the modifications to a brand new file so we don't overwrite your source corpus
+doc.save(output_path)
+doc.close()
+
+print(f"Saved visual debug map to: {output_path}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/dictionary_text_extract.py

@@ -0,0 +1,38 @@
+import subprocess
+import time
+
+# Define the command as a list of arguments
+command = [
+    "docling", 
+    "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf_chunks/chunk_1-001-005.pdf"
+]
+
+print("Running docling command...")
+
+# Record the start time
+start_time = time.perf_counter()
+
+try:
+    # Run the command and wait for it to complete
+    # text=True and capture_output=True lets you capture the terminal output if needed
+    result = subprocess.run(command, capture_output=True, text=True, check=True)
+    
+    # Record the end time
+    end_time = time.perf_counter()
+    
+    # Calculate total duration
+    execution_time = end_time - start_time
+    
+    print("\n--- Command Executed Successfully ---")
+    print(f"Execution Time: {execution_time:.4f} seconds")
+    
+    # Optional: Print the output from docling if you want to see it
+    # print("\nOutput:")
+    # print(result.stdout)
+
+except subprocess.CalledProcessError as e:
+    end_time = time.perf_counter()
+    print("\n--- Command Failed ---")
+    print(f"Execution Time until failure: {end_time - start_time:.4f} seconds")
+    print(f"Error Code: {e.returncode}")
+    print(f"Error Message:\n{e.stderr}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/extract_embedd_toc.py

@@ -0,0 +1,125 @@
+import json
+import os
+from pypdf import PdfReader
+
+# Configuration
+INPUT_PDF_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/MOR.pdf" # Update to your master textbook PDF path if different
+OUTPUT_JSON_PATH = "toc_structure.json"
+
+# Master Section boundaries provided by you
+SECTION_BOUNDARIES = [
+    {"section": 1, "start": 28, "end": 48, "title": "SECTION I: GENERAL"},
+    {"section": 2, "start": 52, "end": 101, "title": "SECTION II: EXAMINATION"},
+    {"section": 3, "start": 104, "end": 168, "title": "SECTION III: APPROACH TO THE PATIENT"},
+    {"section": 4, "start": 172, "end": 259, "title": "SECTION IV: DRUGS"},
+    {"section": 5, "start": 262, "end": 298, "title": "SECTION V: DISEASE OUTCOME MEASURES"},
+    {"section": 6, "start": 302, "end": 383, "title": "SECTION VI: IMAGING INVESTIGATIONS AND JOINT INJECTIONS"},
+    {"section": 7, "start": 386, "end": 422, "title": "SECTION VII: LABORATORY INVESTIGATIONS"},
+    {"section": 8, "start": 426, "end": 486, "title": "SECTION VIII: RHEUMATOID ARTHRITIS, SPONDYLOARTHRITIS, LUPUS, APS, AND SJÖGREN’S SYNDROME"},
+    {"section": 9, "start": 490, "end": 540, "title": "SECTION IX: SYSTEMIC SCLEROSIS, MYOSITIS, MCTD, AND VASCULITIS"},
+    {"section": 10, "start": 544, "end": 586, "title": "SECTION X: CRYSTAL ARTHRITIDES, SARCOIDOSIS, OSTEOARTHRITIS, OSTEOPOROSIS, JIA, AOSD, MACROPHAGE ACTIVATION SYNDROME, AND SEPTIC ARTHRITIS"},
+    {"section": 11, "start": 590, "end": 635, "title": "SECTION XI: ARTHRITIS IN SYSTEMIC DISEASES AND INFECTIONS, RELAPSING POLYCHONDRITIS, OPTICOSPINAL SYNDROME, IPAF, AND PAH IN CTDs"},
+    {"section": 12, "start": 638, "end": 672, "title": "SECTION XII: KAWASAKI DISEASE, PRIMARY IMMUNODEFICIENCY DISEASES, AUTOINFLAMMATORY SYNDROMES, AMYLOIDOSIS, PANNICULITIS, AND SOFT TISSUE RHEUMATISM"}
+]
+
+def get_section_for_page(page_num):
+    """Finds which section dictionary a specific page number belongs to."""
+    for sec in SECTION_BOUNDARIES:
+        if sec["start"] <= page_num <= sec["end"]:
+            return sec
+    return None
+
+def extract_toc_to_json(pdf_path, output_json):
+    if not os.path.exists(pdf_path):
+        print(f"Error: PDF file not found at {pdf_path}")
+        return
+
+    print("Reading PDF outlines...")
+    reader = PdfReader(pdf_path)
+    
+    try:
+        outline = reader.outline
+        if not outline:
+            print("No embedded outline tree found in this PDF file.")
+            return
+    except Exception as e:
+        print(f"Failed to read outline: {e}")
+        return
+
+    # Flatten out the nested pypdf outline tree down to pure dictionary components
+    raw_elements = []
+    
+    def walk_tree(items):
+        for item in items:
+            if isinstance(item, list):
+                walk_tree(item)
+            else:
+                # FIX: Resolve the IndirectObject pointer into a clean 0-based integer index
+                page_index = reader.get_destination_page_number(item)
+                
+                if page_index is not None:
+                    # Convert 0-indexed page to absolute 1-based PDF page number
+                    page_1_indexed = page_index + 1
+                    raw_elements.append({
+                        "title": item.title.strip(),
+                        "start_page": page_1_indexed
+                    })
+
+    walk_tree(outline)
+    
+    # Initialize JSON output format
+    structured_json = {"sections": []}
+    for sec in SECTION_BOUNDARIES:
+        structured_json["sections"].append({
+            "section_number": sec["section"],
+            "section_title": sec["title"],
+            "page_range": f"{sec['start']}-{sec['end']}",
+            "subsections": []
+        })
+
+    # Filter and map sub-elements to their parent sections
+    subsections_by_section = {i: [] for i in range(1, 13)}
+    
+    for element in raw_elements:
+        parent_sec = get_section_for_page(element["start_page"])
+        if parent_sec:
+            # Avoid re-adding the main section titles if they exist in the outline tree itself
+            if "SECTION" in element["title"].upper() and ("I" in element["title"] or "X" in element["title"] or "V" in element["title"]):
+                continue
+            subsections_by_section[parent_sec["section"]].append(element)
+
+    # Calculate exact page ranges for each sub-section sequentially
+    for sec in structured_json["sections"]:
+        sec_num = sec["section_number"]
+        sec_end_page = SECTION_BOUNDARIES[sec_num - 1]["end"]
+        subs = subsections_by_section[sec_num]
+        
+        # Sort subsections sequentially by their starting page
+        subs = sorted(subs, key=lambda x: x["start_page"])
+        
+        for idx, sub in enumerate(subs):
+            start = sub["start_page"]
+            
+            # The end page of this subsection is 1 page before the next subsection starts,
+            # or the terminal boundary of the section itself.
+            if idx + 1 < len(subs):
+                end = subs[idx + 1]["start_page"] - 1
+                # Corner case handling if multiple subsections sit on the exact same page
+                if end < start:
+                    end = start
+            else:
+                end = sec_end_page
+                
+            sec["subsections"].append({
+                "title": sub["title"],
+                "page_range": f"{start}-{end}"
+            })
+
+    # Export out cleanly formatted JSON data
+    with open(output_json, "w", encoding="utf-8") as json_file:
+        json.dump(structured_json, json_file, indent=4, ensure_ascii=False)
+        
+    print(f"Success! Structural mapping exported to: {output_json}")
+
+if __name__ == "__main__":
+    extract_toc_to_json(INPUT_PDF_PATH, OUTPUT_JSON_PATH)

workspace/sprint_1_2/CODEBASE/knowledge/tests/extract_ner_corpus.py

@@ -0,0 +1,46 @@
+
+from pathlib import Path
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.pipeline.vlm_pipeline import VlmPipeline
+from docling.datamodel.pipeline_options import (
+    VlmPipelineOptions,
+)
+from docling.datamodel import vlm_model_specs
+
+pipeline_options = VlmPipelineOptions(
+    vlm_options=vlm_model_specs.SMOLDOCLING_MLX,  # <-- change the model here
+)
+
+converter = DocumentConverter(
+    format_options={
+        InputFormat.PDF: PdfFormatOption(
+            pipeline_cls=VlmPipeline,
+            pipeline_options=pipeline_options,
+        ),
+    }
+)
+
+script_dir = Path(__file__).resolve().parent.parent
+file_dir = script_dir / "corpus" / "indexs" / "mor_index.pdf"
+output_path = script_dir / "corpus" / "indexs" / "mor_index.md"
+doc = converter.convert(source=file_dir).document
+
+# Extract text content
+text_content = doc.export_to_markdown()
+
+# save to markdown
+output_path.parent.mkdir(parents=True, exist_ok=True)
+with open(output_path, "w", encoding="utf-8") as f:
+    f.write(text_content)
+
+
+
+
+
+
+
+
+
+
+

workspace/sprint_1_2/CODEBASE/knowledge/tests/fast_emb_gemma_install.py

@@ -0,0 +1,45 @@
+import os
+import shutil
+from pathlib import Path
+from fastembed.text.onnx_embedding import OnnxTextEmbedding
+from fastembed.common.model_description import PoolingType
+
+# 1. Define a brand new, completely flat directory outside the messy cache
+BASE_DIR = "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/tests/gemma_emb"
+CLEAN_MODEL_DIR = os.path.join(BASE_DIR, "gemma_final")
+os.makedirs(CLEAN_MODEL_DIR, exist_ok=True)
+
+# 2. Source path where Hugging Face put the files based on your error log
+HF_SNAPSHOT_DIR = os.path.join(BASE_DIR, "models--onnx-community--embeddinggemma-300m-ONNX", "snapshots", "5090578d9565bb06545b4552f76e6bc2c93e4a66")
+
+print("Copying files to a clean, flat directory...")
+files_to_copy = [
+    (os.path.join(HF_SNAPSHOT_DIR, "onnx", "model_fp16.onnx"), "model_fp16.onnx"),
+    (os.path.join(HF_SNAPSHOT_DIR, "onnx", "model_fp16.onnx_data"), "model_fp16.onnx_data"),
+    (os.path.join(HF_SNAPSHOT_DIR, "tokenizer.json"), "tokenizer.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "tokenizer_config.json"), "tokenizer_config.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "special_tokens_map.json"), "special_tokens_map.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "config.json"), "config.json"),
+]
+
+for src, filename in files_to_copy:
+    dest = os.path.join(CLEAN_MODEL_DIR, filename)
+    if os.path.exists(src) and not os.path.exists(dest):
+        # Using shutil.copy resolves symlinks automatically, copying the real data!
+        shutil.copy(src, dest)
+        print(f"✓ Copied {filename}")
+
+# 3. Initialize directly using the clean, flat folder
+print("\nInitializing Gemma directly via OnnxEmbedding from clean folder...")
+model = OnnxTextEmbedding(
+    model_dir=Path(CLEAN_MODEL_DIR),
+    model_file="model_fp16.onnx", 
+    pooling=PoolingType.MEAN,
+    max_length=512,
+    threads=2
+)
+
+# 4. Verify it works!
+documents = ["Testing local Gemma embedding generation without cache bugs."]
+embeddings = list(model.embed(documents))
+print(f"\n🎉 Success! Generated vector shape: {embeddings[0].shape}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/ingestion_corpus_db.py

@@ -0,0 +1,99 @@
+import asyncio
+import concurrent.futures
+import time
+import aiosqlite
+from docling.document_converter import DocumentConverter
+import re
+# 1. Global Cache: Load docling models ONCE to maximize M1 efficiency
+print("Initializing Docling models into memory...")
+doc_converter = DocumentConverter() 
+print("Models loaded and ready!")
+
+DB_NAME = "tny_ingestion_corpus.db"
+
+TARGET = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/batches"
+
+def get_pdf_files(target_dir):
+    import os
+    pdf_files = []
+    for root, dirs, files in os.walk(target_dir):
+        for file in files:
+            if file.lower().endswith('.pdf'):
+                pdf_files.append(os.path.join(root, file))
+    # Helper function to sort files naturally (e.g., chunk_2 before chunk_10)
+    def natural_sort_key(s):
+        return [int(text) if text.isdigit() else text.lower() for text in re.split(r'(\d+)', s)]
+    return sorted(pdf_files, key=natural_sort_key)
+
+# database initialization
+async def init_db():
+    async with aiosqlite.connect(DB_NAME) as db:
+        await db.execute("""
+            CREATE TABLE IF NOT EXISTS processed_chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_path TEXT,
+                markdown_content TEXT,
+                execution_time REAL,
+                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        await db.commit()
+    print(f"Database '{DB_NAME}' initialized successfully.")
+
+# Heavy CPU/GPU bound worker function (Synchronous context)
+def process_pdf_worker(file_path: str):
+    start = time.perf_counter()
+    
+    # Run the conversion using the pre-loaded global converter
+    result = doc_converter.convert(file_path)
+    markdown_output = result.document.export_to_markdown()
+        
+    end = time.perf_counter()
+    return markdown_output, (end - start)
+
+
+async def save_to_db(file_path: str, markdown_content: str, exec_time: float):
+    print(f"[Database]: Writing results for {file_path.split('/')[-1]} to SQLite...")
+    async with aiosqlite.connect(DB_NAME) as db:
+        await db.execute(
+            "INSERT INTO processed_chunks (source_path, markdown_content, execution_time) VALUES (?, ?, ?)",
+            (file_path, markdown_content, exec_time)
+        )
+        await db.commit()
+    print(f"[Database]: Write complete for {file_path.split('/')[-1]}!")
+
+async def main():
+    await init_db()
+
+    MAX_M1_WORKERS = 2
+    loop = asyncio.get_running_loop()
+    
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_M1_WORKERS) as pool:
+        print("\n--- Starting Ingestion System ---")
+        files_to_ingest = get_pdf_files(TARGET)
+        
+        # 1. Create a helper async function to handle the processing + saving pipeline per file
+        async def pipeline_worker(file_path):
+            # Hand off the heavy Docling job to the pool
+            markdown_content, execution_time = await loop.run_in_executor(pool, process_pdf_worker, file_path)
+            print(f"[Background Worker]: Docling extraction finished for {file_path.split('/')[-1]} in {execution_time:.2f}s")
+            
+            # Save it to the DB as soon as it's done
+            await save_to_db(file_path, markdown_content, execution_time)
+
+        # 2. Fire off ALL tasks into the background immediately (Non-blocking loop)
+        tasks = []
+        for file_path in files_to_ingest:
+            print(f"[Ingest API]: Queueing {file_path.split('/')[-1]}...")
+            tasks.append(pipeline_worker(file_path))
+        
+        # 3. NOW await them all together. This forces the ThreadPoolExecutor 
+        # to actually process 2 files at the exact same time!
+        await asyncio.gather(*tasks)
+
+    print("\n--- All files processed concurrently! ---")
+
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

workspace/sprint_1_2/CODEBASE/knowledge/tests/markdown_chunking_oho.py

@@ -0,0 +1,17 @@
+from langchain_text_splitters import MarkdownHeaderTextSplitter
+from pathlib import Path
+
+oho_markdown_file = Path(__file__).resolve().parent.parent.parent.parent / "CODEBASE" / "knowledge" / "corpus" / "indexs" / "oho_index.md"
+with open(oho_markdown_file, "r", encoding="utf-8") as f:
+    text = f.read()
+
+oho_splitter = MarkdownHeaderTextSplitter(headers_to_split_on = [
+    ("##", "id"), # split based on the alphabetical headers (##) and assign the id to the chunk
+])
+chunks = oho_splitter.split_text(text)
+
+print(f"Total chunks created: {len(chunks)}")
+print(f"First chunk: {chunks[1]}") # start from the 1 to 16
+for i, chunk in enumerate(chunks[1:17], 1):
+    print(f"Chunk {i} : {chunk.page_content}")
+    print("\n \n")

workspace/sprint_1_2/CODEBASE/knowledge/tests/mor_split_page.py

@@ -0,0 +1,96 @@
+import json
+import os
+import re
+import subprocess
+
+# Paths
+JSON_INPUT_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/toc_structure.json"
+MASTER_PDF_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/mor_bounded.pdf"
+OUTPUT_DIR = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/sections"
+
+def format_page_number(page_str):
+    """Converts a page string to an integer and pads it to a 3-digit string (e.g., 28 -> '028')."""
+    return f"{int(page_str):03d}"
+
+def clean_filename_part(text):
+    """Extracts section numbers cleanly (e.g., '1.1' from '1.1 History Taking') or replaces bad characters."""
+    # Try to find a subsection number at the start of the title like "1.1" or "10.4"
+    match = re.match(r'^(\d+\.\d+)', text.strip())
+    if match:
+        return match.group(1)
+    
+    # Fallback safe string cleaning if no clean number pattern exists
+    invalid_chars = ['/', '\\', ':', '*', '?', '"', '<', '>', '|', ' ']
+    clean_text = text.strip()
+    for char in invalid_chars:
+        clean_text = clean_text.replace(char, '_')
+    return clean_text[:20]
+
+def split_pdf_by_subsections():
+    # Ensure output folder exists
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    if not os.path.exists(JSON_INPUT_PATH):
+        print(f"Error: JSON metadata file not found at {JSON_INPUT_PATH}")
+        return
+        
+    with open(JSON_INPUT_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    print("Loaded TOC configurations. Slicing down to SUBSECTION units...\n")
+
+    for sec in data.get("sections", []):
+        sec_num = sec["section_number"]
+        subsections = sec.get("subsections", [])
+        
+        if not subsections:
+            print(f"[Notice] Section {sec_num} has no nested subsections. Skipping.")
+            continue
+
+        print(f"Processing Section {sec_num} -> Found {len(subsections)} subsections to slice:")
+
+        for sub in subsections:
+            title = sub["title"]             # Matches: "1.1 History Taking..."
+            page_range = sub["page_range"]   # Matches: "28-32"
+            
+            try:
+                raw_start, raw_end = page_range.split("-")
+                padded_start = format_page_number(raw_start)
+                padded_end = format_page_number(raw_end)
+            except (ValueError, AttributeError):
+                print(f"  └─ [Warning] Skipping '{title}': Invalid range format '{page_range}'.")
+                continue
+            
+            # Extract a clean identifier prefix like 'sub_1.1' or 'sub_1.2'
+            sub_id = clean_filename_part(title)
+            
+            # Build your targeted filename pattern: sub_1.1_chunk-028-032.pdf
+            output_filename = f"sub_{sub_id}_chunk-{padded_start}-{padded_end}.pdf"
+            output_path = os.path.join(OUTPUT_DIR, output_filename)
+
+            # Formulate the execution payload for qpdf
+            command = [
+                "qpdf",
+                MASTER_PDF_PATH,
+                "--pages",
+                ".",
+                page_range,
+                "--",
+                output_path
+            ]
+
+            try:
+                subprocess.run(command, check=True, capture_output=True, text=True)
+                print(f"  └─ Generated: {output_filename} ({page_range})")
+            except subprocess.CalledProcessError as e:
+                print(f"  └─ [Error] qpdf failed to slice subsection '{title}'.")
+                print(f"     Details: {e.stderr}")
+                continue
+
+    print("\n--- Subsection granularity slicing complete! Check your output folder. ---")
+
+if __name__ == "__main__":
+    if not os.path.exists(MASTER_PDF_PATH):
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'. Please verify your configurations.")
+    else:
+        split_pdf_by_subsections()

workspace/sprint_1_2/CODEBASE/knowledge/tests/oho_split_page.py

@@ -0,0 +1,132 @@
+import re
+import subprocess
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+INDEX_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "index.txt"
+MASTER_PDF_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "oho_bounded.pdf"
+OUTPUT_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "sections"
+BATCH_SIZE = 5
+
+SECTION_LINE_PATTERN = re.compile(
+    r"^\s*section\s+(\d+)\s*:\s*(\d+)\s*-\s*(\d+)\s*$",
+    re.IGNORECASE,
+)
+
+
+def format_page_number(page: int) -> str:
+    """Pad a page number to three digits (e.g. 32 -> '032')."""
+    return f"{page:03d}"
+
+
+def parse_index_file(index_path: Path) -> list[dict[str, int]]:
+    """Parse section page ranges from index.txt."""
+    sections: list[dict[str, int]] = []
+
+    with open(index_path, "r", encoding="utf-8") as f:
+        for line_num, line in enumerate(f, start=1):
+            match = SECTION_LINE_PATTERN.match(line)
+            if not match:
+                if line.strip():
+                    print(f"[Warning] Skipping unrecognized line {line_num}: {line.strip()}")
+                continue
+
+            section_num, start_page, end_page = match.groups()
+            sections.append(
+                {
+                    "section_number": int(section_num),
+                    "start_page": int(start_page),
+                    "end_page": int(end_page),
+                }
+            )
+
+    return sections
+
+
+def iter_page_batches(start_page: int, end_page: int, batch_size: int = BATCH_SIZE):
+    """Yield consecutive page batches within a section range."""
+    batch_num = 1
+    current = start_page
+
+    while current <= end_page:
+        batch_end = min(current + batch_size - 1, end_page)
+        yield batch_num, current, batch_end
+        batch_num += 1
+        current = batch_end + 1
+
+
+def run_qpdf_slice(page_range: str, output_path: Path) -> None:
+    command = [
+        "qpdf",
+        str(MASTER_PDF_PATH),
+        "--pages",
+        ".",
+        page_range,
+        "--",
+        str(output_path),
+    ]
+    subprocess.run(command, check=True, capture_output=True, text=True)
+
+
+def split_pdf_by_section_batches() -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not INDEX_PATH.exists():
+        print(f"Error: Index file not found at {INDEX_PATH}")
+        return
+
+    sections = parse_index_file(INDEX_PATH)
+    if not sections:
+        print(f"Error: No valid section entries found in {INDEX_PATH}")
+        return
+
+    print(
+        f"Loaded {len(sections)} section ranges from index.txt. "
+        f"Slicing each section into batches of {BATCH_SIZE} pages...\n"
+    )
+
+    total_batches = 0
+
+    for section in sections:
+        section_num = section["section_number"]
+        start_page = section["start_page"]
+        end_page = section["end_page"]
+        batches = list(iter_page_batches(start_page, end_page))
+
+        print(
+            f"Processing Section {section_num} "
+            f"({start_page}-{end_page}) -> {len(batches)} batch(es):"
+        )
+
+        for batch_num, batch_start, batch_end in batches:
+            page_range = f"{batch_start}-{batch_end}"
+            padded_start = format_page_number(batch_start)
+            padded_end = format_page_number(batch_end)
+            output_filename = (
+                f"sec_{section_num}_batch-{batch_num:02d}_"
+                f"chunk-{padded_start}-{padded_end}.pdf"
+            )
+            output_path = OUTPUT_DIR / output_filename
+
+            try:
+                run_qpdf_slice(page_range, output_path)
+                print(f"  └─ Generated: {output_filename} ({page_range})")
+                total_batches += 1
+            except subprocess.CalledProcessError as e:
+                print(
+                    f"  └─ [Error] qpdf failed to slice section {section_num}, "
+                    f"batch {batch_num}."
+                )
+                print(f"     Details: {e.stderr}")
+
+    print(
+        f"\n--- Section batch slicing complete! "
+        f"Generated {total_batches} PDF(s). Check {OUTPUT_DIR}. ---"
+    )
+
+
+if __name__ == "__main__":
+    if not MASTER_PDF_PATH.exists():
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'.")
+    else:
+        split_pdf_by_section_batches()

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_assemble.py

@@ -0,0 +1,20 @@
+import asyncio
+from pathlib import Path
+from implementation.ingestion.assemble import assemble_logical_units
+from implementation.ingestion.config import resolve_corpus_db_path
+from implementation.ingestion.sqlite_store import load_processed_chunks
+
+async def inspect_assembly(book_id: str = "mor"):
+    db_path = resolve_corpus_db_path(book_id)
+    rows = await load_processed_chunks(db_path)  # list of (id, source_path, markdown_content)
+    units = assemble_logical_units(rows, book_id=book_id)
+    print(f"processed_chunks rows: {len(rows)}")
+    print(f"logical units:         {len(units)}")
+    for unit in units[:10]:
+        print(f"\n--- {unit.logical_unit_id} ---")
+        print(f"pages: {unit.page_start}-{unit.page_end}")
+        print(f"parent_title: {unit.parent_title}")
+        print(f"source_ids: {unit.source_extraction_ids}")
+        print(f"content preview: {unit.content}...")
+
+asyncio.run(inspect_assembly("mor"))

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_bert_ner.py

@@ -0,0 +1,31 @@
+from gliner import GLiNER
+
+model = GLiNER.from_pretrained("gliner-community/gliner_small-v2.5",
+                                map_location="cpu",
+                                cache_dir="/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/tests",
+                                dtype="fp16")
+
+# 1. Define your MSK-domain text snippet (e.g., from a radiology or orthopedic report)
+text = """
+The patient presents with severe lower back pain radiating down the left leg. 
+An MRI of the lumbar spine reveals a moderate L4-L5 disc herniation causing 
+mild stenosis of the neural foramina. We discussed proceeding with a 
+lumbar epidural steroid injection for pain management.
+"""
+
+# 2. Define domain-specific MSK clinical labels
+labels = [
+    "Anatomical_Structure",  # e.g., lumbar spine, L4-L5, neural foramina
+    "Pathology_Condition",   # e.g., disc herniation, stenosis
+    "Symptom",               # e.g., lower back pain
+    "Procedure_Treatment"    # e.g., MRI, lumbar epidural steroid injection
+]
+
+# 3. Predict entities using your model
+entities = model.inference(text, labels, threshold=0.5)
+
+# 4. Print the extracted MSK clinical entities
+print("Extracted MSK Entities:")
+print("-" * 30)
+for entity in entities:
+    print(f"{entity['text']} => {entity['label']}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_chunker.py

@@ -0,0 +1,137 @@
+"""Unit tests for prose/table block extraction and chunking."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from implementation.ingestion.assemble import LogicalUnit
+from implementation.ingestion.chunker import (
+    chunk_logical_unit,
+    embedding_text,
+    extract_blocks,
+)
+from implementation.ingestion.config import DEFAULT_CHUNKER_CONFIG
+
+
+@dataclass
+class _MockEmbedder:
+    """Lightweight embedder stub for chunking tests."""
+
+    def count_tokens(self, text: str) -> int:
+        return max(1, len(text.split()))
+
+    def embed_clustering(self, text: str) -> list[float]:
+        return [float(len(text)), 1.0, 0.5]
+
+    def embed_document(self, text: str, *, title: str | None = None) -> list[float]:
+        return [float(len(text)), 1.0, 0.5]
+
+
+def _sample_unit(content: str, book_id: str = "mor") -> LogicalUnit:
+    return LogicalUnit(
+        logical_unit_id="sub_1.1",
+        book_id=book_id,
+        content=content,
+        source_extraction_ids=[1],
+        page_start=1,
+        page_end=2,
+        subsection_id="1.1",
+        parent_title="Sample subsection",
+    )
+
+
+def test_extract_blocks_splits_prose_and_wrapped_table():
+    text = (
+        "Juvenile SLE presents with fever and rash.\n\n"
+        "Table 1. Clinical features\n\n"
+        "| Feature | Frequency |\n"
+        "| --- | --- |\n"
+        "| Fever | High |\n"
+        "Treatment depends on severity."
+    )
+
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 3
+    assert blocks[0].block_type == "prose"
+    assert "Juvenile SLE" in blocks[0].text
+    assert "Table 1." not in blocks[0].text
+    assert blocks[1].block_type == "table"
+    assert blocks[1].text.startswith("Table 1. Clinical features")
+    assert "<table>" in blocks[1].text
+    assert blocks[2].block_type == "prose"
+    assert "Treatment depends" in blocks[2].text
+
+
+def test_extract_blocks_prose_only_when_no_tables():
+    text = "Only prose here with no tabular content."
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 1
+    assert blocks[0].block_type == "prose"
+    assert blocks[0].text == text
+
+
+def test_chunk_logical_unit_emits_separate_table_chunk():
+    unit = _sample_unit(
+        "Intro paragraph about dosing.\n\n"
+        "Table 2. Dosing guide\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>\n\n"
+        "Follow-up monitoring is required."
+    )
+
+    records = chunk_logical_unit(unit, embedder=_MockEmbedder(), config=DEFAULT_CHUNKER_CONFIG)
+
+    assert len(records) == 3
+    assert "Intro paragraph" in records[0].content
+    assert "<table>" in records[1].content
+    assert "Table 2. Dosing guide" in records[1].content
+    assert "Follow-up monitoring" in records[2].content
+    assert records[0].chunk_index == 0
+    assert records[1].chunk_index == 1
+    assert records[2].chunk_index == 2
+    assert records[1].embed_content is not None
+    assert "Sample subsection" not in records[1].embed_content
+    assert "<table>" in records[1].embed_content
+
+
+def test_embedding_text_returns_table_body_without_title():
+    unit = _sample_unit(
+        "Table 2. Dosing guide\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>"
+    )
+    records = chunk_logical_unit(unit, embedder=_MockEmbedder(), config=DEFAULT_CHUNKER_CONFIG)
+    table_record = records[0]
+    table_record = table_record.__class__(**{**table_record.__dict__, "embed_content": None})
+
+    rebuilt = embedding_text(table_record)
+
+    assert "Sample subsection" not in rebuilt
+    assert "<table>" in rebuilt
+
+
+def test_extract_blocks_skips_wrap_when_tables_already_normalized():
+    text = (
+        "Intro paragraph.\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>\n\n"
+        "Follow-up text."
+    )
+
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 3
+    assert blocks[1].block_type == "table"
+    assert blocks[1].text.count("<table>") == 1
+    assert blocks[1].text.count("</table>") == 1

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_embedding.py

@@ -0,0 +1,27 @@
+"""Unit tests for task-routed EmbeddingGemma prefixes."""
+
+from implementation.ingestion.embedding import EmbedTask, format_embed_input
+
+
+def test_retrieval_document_uses_parent_title_in_prefix():
+    result = format_embed_input(
+        "Chunk body text.",
+        EmbedTask.RETRIEVAL_DOCUMENT,
+        title="Juvenile SLE",
+    )
+    assert result == "title: Juvenile SLE | text: Chunk body text."
+
+
+def test_retrieval_document_falls_back_to_none_title():
+    result = format_embed_input("Chunk body.", EmbedTask.RETRIEVAL_DOCUMENT)
+    assert result == "title: none | text: Chunk body."
+
+
+def test_retrieval_query_uses_search_prefix():
+    result = format_embed_input("pediatric dosing", EmbedTask.RETRIEVAL_QUERY)
+    assert result == "task: search result | query: pediatric dosing"
+
+
+def test_clustering_uses_clustering_prefix():
+    result = format_embed_input("Sentence one.", EmbedTask.CLUSTERING)
+    assert result == "task: clustering | query: Sentence one."

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_normalize_newlines.py

@@ -0,0 +1,39 @@
+"""Unit tests for assembled-text newline normalization."""
+
+from implementation.ingestion.assemble import _clean_assembled_text, _normalize_newlines
+
+
+def test_joins_soft_wrapped_paragraph_lines():
+    raw = "He was unable to raise both arms due to\n\nsevere shoulder pain."
+    assert "due to severe shoulder pain" in _normalize_newlines(raw)
+    assert "due to\n\nsevere" not in _normalize_newlines(raw)
+
+
+def test_preserves_markdown_list_lines():
+    raw = "- Spiral fracture : gãy xoắn\n- Communited fracture : gãy nát"
+    normalized = _normalize_newlines(raw)
+    assert normalized.count("\n") == 1
+    assert "- Spiral fracture" in normalized
+    assert "- Communited fracture" in normalized
+
+
+def test_collapses_repeated_spaces():
+    raw = "There  was  no  loss  consciousness"
+    assert _normalize_newlines(raw) == "There was no loss consciousness"
+
+
+def test_collapses_excess_blank_lines():
+    raw = "Line one\n\n\n\nLine two"
+    assert _normalize_newlines(raw) == "Line one\n\nLine two"
+
+
+def test_joins_hyphenation_breaks():
+    raw = "medi-\ncal terminology"
+    assert _normalize_newlines(raw) == "medical terminology"
+
+
+def test_clean_assembled_text_strips_image_markers_and_normalizes():
+    raw = "<!-- image -->\nHe was thrown\n\nfrom his motorcycle."
+    cleaned = _clean_assembled_text(raw)
+    assert "<!-- image -->" not in cleaned
+    assert "thrown from his motorcycle" in cleaned

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_tny_md_processed_chunks.py

@@ -0,0 +1,45 @@
+"""Tests for tny.md → processed_chunks structural rebuild."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tny_md_processed_chunks import (
+    TNY_MD_PATH,
+    recursive_structure_chunk,
+    validate_source_paths,
+    wrap_tables,
+)
+
+
+def test_wrap_tables_adds_table_tags():
+    sample = "| A | B |\n| --- | --- |\n| 1 | 2 |"
+    wrapped = wrap_tables(sample)
+    assert wrapped.startswith("<table>")
+    assert wrapped.endswith("</table>")
+
+
+def test_tny_md_produces_structural_chunks():
+    if not TNY_MD_PATH.exists():
+        pytest.skip(f"missing fixture: {TNY_MD_PATH}")
+
+    text = TNY_MD_PATH.read_text(encoding="utf-8")
+    chunks = recursive_structure_chunk(text)
+
+    assert len(chunks) >= 10
+    assert any(chunk.block_type == "table" for chunk in chunks)
+    assert all(chunk.markdown_content.strip() for chunk in chunks)
+    validate_source_paths(chunks)
+
+
+def test_tny_md_table_chunk_contains_joint_classification():
+    if not TNY_MD_PATH.exists():
+        pytest.skip(f"missing fixture: {TNY_MD_PATH}")
+
+    chunks = recursive_structure_chunk(TNY_MD_PATH.read_text(encoding="utf-8"))
+    table_chunks = [c for c in chunks if c.block_type == "table"]
+    assert len(table_chunks) == 1
+    assert "Fibrous" in table_chunks[0].markdown_content
+    assert "<table>" in table_chunks[0].markdown_content

workspace/sprint_1_2/CODEBASE/knowledge/tests/tny_md_processed_chunks.py

@@ -0,0 +1,373 @@
+"""Rebuild TNY processed_chunks from corpus/pdf/tny/tny.md.
+
+Pipeline:
+  1. Wrap pipe markdown tables in <table>...</table>
+  2. Recursively split on section (##), subsection (###), sub-subsection (####)
+  3. At each leaf, split prose vs table blocks
+  4. Replace all rows in tny_ingestion_corpus.db processed_chunks
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import re
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+if str(KNOWLEDGE_ROOT) not in sys.path:
+    sys.path.insert(0, str(KNOWLEDGE_ROOT))
+
+import aiosqlite
+
+from implementation.ingestion.assemble import _wrap_markdown_tables
+from implementation.ingestion.chunker import extract_blocks
+from implementation.ingestion.config import resolve_corpus_db_path
+from implementation.ingestion.metadata_parser import parse_source_path
+
+TNY_MD_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "tny.md"
+MD_CHUNKS_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "md_chunks"
+DEBUG_LOG_PATH = (
+    Path(__file__).resolve().parents[6]
+    / ".cursor"
+    / "debug-de2ea8.log"
+)
+
+HEADER_SPLITS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"^(##\s+.+)$", re.MULTILINE), "section"),
+    (re.compile(r"^(###\s+.+)$", re.MULTILINE), "subsection"),
+    (re.compile(r"^(####\s+.+)$", re.MULTILINE), "subsubsection"),
+]
+
+
+@dataclass
+class StructuralChunk:
+    """One processed_chunks row candidate derived from tny.md."""
+
+    markdown_content: str
+    block_type: str
+    section_title: str | None = None
+    subsection_title: str | None = None
+    subsubsection_title: str | None = None
+    ancestry: list[str] = field(default_factory=list)
+
+
+# #region agent log
+def _debug_log(
+    hypothesis_id: str,
+    location: str,
+    message: str,
+    data: dict | None = None,
+    run_id: str = "initial",
+) -> None:
+    payload = {
+        "sessionId": "de2ea8",
+        "runId": run_id,
+        "hypothesisId": hypothesis_id,
+        "location": location,
+        "message": message,
+        "data": data or {},
+        "timestamp": int(time.time() * 1000),
+    }
+    with DEBUG_LOG_PATH.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
+
+
+# #endregion
+
+
+def wrap_tables(text: str) -> str:
+    """Step 1: wrap pipe markdown tables in <table> tags."""
+    wrapped = _wrap_markdown_tables(text)
+    # #region agent log
+    _debug_log(
+        "H1",
+        "tny_md_processed_chunks.py:wrap_tables",
+        "table wrap complete",
+        {
+            "input_len": len(text),
+            "output_len": len(wrapped),
+            "table_tag_count": wrapped.count("<table>"),
+        },
+    )
+    # #endregion
+    return wrapped
+
+
+def _header_title(line: str) -> str:
+    return re.sub(r"^#{1,6}\s+", "", line.strip())
+
+
+def _split_on_header_level(
+    text: str,
+    level: int,
+    ancestry: list[str],
+) -> list[StructuralChunk]:
+    """Recursively split markdown by section / subsection / sub-subsection."""
+    text = text.strip()
+    if not text:
+        return []
+
+    if level >= len(HEADER_SPLITS):
+        return _split_leaf_blocks(text, ancestry)
+
+    pattern, kind = HEADER_SPLITS[level]
+    matches = list(pattern.finditer(text))
+    if not matches:
+        # #region agent log
+        _debug_log(
+            "H3",
+            "tny_md_processed_chunks.py:_split_on_header_level",
+            "no headers at level; descend",
+            {"level": level, "kind": kind, "text_len": len(text)},
+        )
+        # #endregion
+        return _split_on_header_level(text, level + 1, ancestry)
+
+    chunks: list[StructuralChunk] = []
+    if matches[0].start() > 0:
+        preamble = text[: matches[0].start()].strip()
+        if preamble:
+            chunks.extend(_split_on_header_level(preamble, level + 1, ancestry))
+
+    for index, match in enumerate(matches):
+        start = match.start()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+        section_text = text[start:end].strip()
+        if not section_text:
+            continue
+
+        title = _header_title(match.group(1))
+        next_ancestry = [*ancestry, title]
+        child_chunks = _split_on_header_level(section_text, level + 1, next_ancestry)
+        for chunk in child_chunks:
+            if kind == "section":
+                chunk.section_title = title
+            elif kind == "subsection":
+                chunk.subsection_title = title
+            elif kind == "subsubsection":
+                chunk.subsubsection_title = title
+        chunks.extend(child_chunks)
+
+    return chunks
+
+
+def _split_leaf_blocks(text: str, ancestry: list[str]) -> list[StructuralChunk]:
+    """Split a leaf node into prose and table processed_chunks rows."""
+    blocks = extract_blocks(text)
+    if not blocks:
+        return []
+
+    section_title = ancestry[0] if ancestry else None
+    subsection_title = ancestry[1] if len(ancestry) > 1 else None
+    subsubsection_title = ancestry[2] if len(ancestry) > 2 else None
+
+    chunks: list[StructuralChunk] = []
+    for block in blocks:
+        content = block.text.strip()
+        if not content:
+            continue
+        chunks.append(
+            StructuralChunk(
+                markdown_content=content,
+                block_type=block.block_type,
+                section_title=section_title,
+                subsection_title=subsection_title,
+                subsubsection_title=subsubsection_title,
+                ancestry=list(ancestry),
+            )
+        )
+
+    # #region agent log
+    _debug_log(
+        "H2",
+        "tny_md_processed_chunks.py:_split_leaf_blocks",
+        "leaf blocks extracted",
+        {
+            "ancestry": ancestry,
+            "block_count": len(chunks),
+            "block_types": [c.block_type for c in chunks],
+        },
+    )
+    # #endregion
+    return chunks
+
+
+def recursive_structure_chunk(text: str) -> list[StructuralChunk]:
+    """Step 2: recursive structural split with table isolation."""
+    wrapped = wrap_tables(text)
+    chunks = _split_on_header_level(wrapped, level=0, ancestry=[])
+    non_empty = [chunk for chunk in chunks if chunk.markdown_content.strip()]
+    # #region agent log
+    _debug_log(
+        "H2",
+        "tny_md_processed_chunks.py:recursive_structure_chunk",
+        "recursive split summary",
+        {
+            "total_chunks": len(non_empty),
+            "prose_count": sum(1 for c in non_empty if c.block_type == "prose"),
+            "table_count": sum(1 for c in non_empty if c.block_type == "table"),
+        },
+    )
+    # #endregion
+    return non_empty
+
+
+def _slugify(value: str, max_len: int = 40) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
+    return (slug[:max_len] or "chunk").strip("_")
+
+
+def build_source_path(index: int, chunk: StructuralChunk) -> str:
+    """Synthetic batch-style path parseable by metadata_parser for TNY."""
+    MD_CHUNKS_DIR.mkdir(parents=True, exist_ok=True)
+    page = index + 1
+    slug_parts = [_slugify(part) for part in chunk.ancestry if part]
+    slug_parts.append(chunk.block_type)
+    slug = "_".join(slug_parts)[:80] or f"{chunk.block_type}_{page}"
+    filename = f"batch-{page:02d}_chunk-{page:03d}-{page:03d}.pdf"
+    return str(MD_CHUNKS_DIR / filename)
+
+
+def validate_source_paths(chunks: list[StructuralChunk]) -> None:
+    """Ensure every synthetic path parses as TNY batch metadata."""
+    paths = [build_source_path(i, chunk) for i, chunk in enumerate(chunks)]
+    for path in paths:
+        parse_source_path(path, book_id="tny")
+    if len(paths) != len(set(paths)):
+        raise ValueError("Duplicate source_path values generated for processed_chunks")
+
+
+async def rebuild_processed_chunks(
+    db_path: Path,
+    chunks: list[StructuralChunk],
+    *,
+    dry_run: bool = False,
+) -> tuple[int, int]:
+    """Step 3: delete old processed_chunks rows and insert markdown-derived rows."""
+    if not dry_run:
+        async with aiosqlite.connect(db_path) as db:
+            cursor = await db.execute("SELECT COUNT(*) FROM processed_chunks")
+            old_count = int((await cursor.fetchone())[0])
+
+            await db.execute("DELETE FROM processed_chunks")
+            deleted = old_count
+
+            inserted = 0
+            for index, chunk in enumerate(chunks):
+                source_path = build_source_path(index, chunk)
+                await db.execute(
+                    """
+                    INSERT INTO processed_chunks (source_path, markdown_content, execution_time)
+                    VALUES (?, ?, ?)
+                    """,
+                    (source_path, chunk.markdown_content, 0.0),
+                )
+                inserted += 1
+            await db.commit()
+
+            cursor = await db.execute("SELECT COUNT(*) FROM processed_chunks")
+            new_count = int((await cursor.fetchone())[0])
+    else:
+        deleted = -1
+        inserted = len(chunks)
+        new_count = len(chunks)
+
+    # #region agent log
+    _debug_log(
+        "H4",
+        "tny_md_processed_chunks.py:rebuild_processed_chunks",
+        "db rebuild complete",
+        {
+            "dry_run": dry_run,
+            "deleted_old_rows": deleted,
+            "inserted_rows": inserted,
+            "final_row_count": new_count,
+        },
+        run_id="post-fix" if not dry_run else "dry-run",
+    )
+    # #endregion
+    return inserted, new_count
+
+
+async def run(
+    md_path: Path = TNY_MD_PATH,
+    book_id: str = "tny",
+    dry_run: bool = False,
+) -> None:
+    if not md_path.exists():
+        raise FileNotFoundError(f"TNY markdown not found: {md_path}")
+
+    raw_text = md_path.read_text(encoding="utf-8")
+    chunks = recursive_structure_chunk(raw_text)
+    validate_source_paths(chunks)
+
+    db_path = resolve_corpus_db_path(book_id)
+    if not db_path.parent.exists():
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS processed_chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_path TEXT,
+                markdown_content TEXT,
+                execution_time REAL,
+                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+            """
+        )
+        await db.commit()
+
+    inserted, final_count = await rebuild_processed_chunks(db_path, chunks, dry_run=dry_run)
+
+    print(f"Source markdown: {md_path}")
+    print(f"Structural chunks: {len(chunks)} (prose={sum(1 for c in chunks if c.block_type == 'prose')}, "
+          f"table={sum(1 for c in chunks if c.block_type == 'table')})")
+    print(f"Database: {db_path}")
+    if dry_run:
+        print("Dry run — no database writes performed.")
+    else:
+        print(f"Replaced processed_chunks: inserted={inserted}, final_count={final_count}")
+
+    for index, chunk in enumerate(chunks[:5], start=1):
+        print(f"\n--- preview {index}/{len(chunks)} [{chunk.block_type}] ---")
+        print(f"ancestry: {' > '.join(chunk.ancestry) or '(root)'}")
+        print(chunk.markdown_content[:200] + ("..." if len(chunk.markdown_content) > 200 else ""))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Rebuild TNY processed_chunks from tny.md with table wrap + structural split.",
+    )
+    parser.add_argument(
+        "--md-path",
+        type=Path,
+        default=TNY_MD_PATH,
+        help="Path to tny.md (default: corpus/pdf/tny/tny.md)",
+    )
+    parser.add_argument(
+        "--book-id",
+        default="tny",
+        help="Book id for checkpoint DB resolution (default: tny)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Split and validate only; do not write SQLite.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    asyncio.run(run(md_path=args.md_path, book_id=args.book_id, dry_run=args.dry_run))
+
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/tests/tny_split_page.py

@@ -0,0 +1,88 @@
+import subprocess
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+MASTER_PDF_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "tny_bounded.pdf"
+OUTPUT_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "batches"
+BATCH_SIZE = 5
+
+
+def format_page_number(page: int) -> str:
+    """Pad a page number to three digits (e.g. 1 -> '001')."""
+    return f"{page:03d}"
+
+
+def get_pdf_page_count(pdf_path: Path) -> int:
+    result = subprocess.run(
+        ["qpdf", "--show-npages", str(pdf_path)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return int(result.stdout.strip())
+
+
+def iter_page_batches(start_page: int, end_page: int, batch_size: int = BATCH_SIZE):
+    """Yield consecutive page batches across the full document."""
+    batch_num = 1
+    current = start_page
+
+    while current <= end_page:
+        batch_end = min(current + batch_size - 1, end_page)
+        yield batch_num, current, batch_end
+        batch_num += 1
+        current = batch_end + 1
+
+
+def run_qpdf_slice(page_range: str, output_path: Path) -> None:
+    command = [
+        "qpdf",
+        str(MASTER_PDF_PATH),
+        "--pages",
+        ".",
+        page_range,
+        "--",
+        str(output_path),
+    ]
+    subprocess.run(command, check=True, capture_output=True, text=True)
+
+
+def split_pdf_into_batches() -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    total_pages = get_pdf_page_count(MASTER_PDF_PATH)
+    batches = list(iter_page_batches(1, total_pages))
+
+    print(
+        f"Loaded '{MASTER_PDF_PATH.name}' ({total_pages} pages). "
+        f"Slicing into batches of {BATCH_SIZE} pages...\n"
+    )
+
+    total_generated = 0
+
+    for batch_num, batch_start, batch_end in batches:
+        page_range = f"{batch_start}-{batch_end}"
+        padded_start = format_page_number(batch_start)
+        padded_end = format_page_number(batch_end)
+        output_filename = f"batch-{batch_num:02d}_chunk-{padded_start}-{padded_end}.pdf"
+        output_path = OUTPUT_DIR / output_filename
+
+        try:
+            run_qpdf_slice(page_range, output_path)
+            print(f"  └─ Generated: {output_filename} ({page_range})")
+            total_generated += 1
+        except subprocess.CalledProcessError as e:
+            print(f"  └─ [Error] qpdf failed to slice batch {batch_num}.")
+            print(f"     Details: {e.stderr}")
+
+    print(
+        f"\n--- Batch slicing complete! "
+        f"Generated {total_generated} PDF(s). Check {OUTPUT_DIR}. ---"
+    )
+
+
+if __name__ == "__main__":
+    if not MASTER_PDF_PATH.exists():
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'.")
+    else:
+        split_pdf_into_batches()

workspace/sprint_1_2/CODEBASE/knowledge/tests/use_gemma.py

@@ -0,0 +1,83 @@
+import torch
+import numpy as np
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+
+LOCAL_MODEL_DIR = "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/implementation/model/gemma_final"
+
+if not hasattr(torch, "int4"):
+    torch.int4 = torch.int8
+
+tokenizer = AutoTokenizer.from_pretrained(LOCAL_MODEL_DIR, local_files_only=True)
+model = ORTModelForFeatureExtraction.from_pretrained(
+    LOCAL_MODEL_DIR, file_name="model_fp16.onnx", local_files_only=True, provider="CPUExecutionProvider", use_io_binding=False
+)
+
+def get_embedding(text, target_dim=256):
+    inputs = tokenizer(text, max_length=512, padding=True, truncation=True, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model(**inputs)
+    raw_states = outputs.last_hidden_state.numpy()
+    mean_pooled = np.mean(raw_states, axis=1)[0]
+    scaled = mean_pooled[:target_dim]
+    return scaled / np.linalg.norm(scaled)
+
+def cosine_similarity(v1, v2):
+    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
+
+# --- TEST DATA ---
+# query_raw = "How do I fix a leaking faucet pipe?"
+# doc_relevant_raw = "To repair a copper pipe valve joint, first isolate the main water supply and drain the lines."
+# doc_irrelevant_raw = "The company announced a 3-for-1 stock split starting next Monday for retail investors."
+
+
+# --- THE HARD NEGATIVE DATA ---
+query_raw = "Apple stock price performance this quarter"
+doc_relevant_raw = "AAPL shares rallied 3% following strong fiscal earnings reports today."
+doc_irrelevant_raw = "The organic Honeycrisp apple crop harvested this quarter yielded excellent crisp fruit."
+
+# 1. UNTUNED TEST (No instructions)
+print("--- RUNNING UNTUNED TEST ---")
+v_query_ut = get_embedding(query_raw)
+v_rel_ut = get_embedding(doc_relevant_raw)
+v_irrel_ut = get_embedding(doc_irrelevant_raw)
+
+sim_rel_ut = cosine_similarity(v_query_ut, v_rel_ut)
+sim_irrel_ut = cosine_similarity(v_query_ut, v_irrel_ut)
+print(f"Untuned Cosine Similarity (Relevant Doc):   {sim_rel_ut:.4f}")
+print(f"Untuned Cosine Similarity (Irrelevant Doc): {sim_irrel_ut:.4f}")
+print(f"Untuned Margin Gap:                         {sim_rel_ut - sim_irrel_ut:.4f}\n")
+
+# --- CORRECTED TUNED TEST (Google Asymmetric Prompt Routing) ---
+print("--- RUNNING CORRECTED TUNED TEST ---")
+
+AVAILABLE_TASK = {
+    "query": "task: search result | query: ",
+    "document": "title: none | text: ",
+    "BitextMining": "task: search result | query: ",
+    "Clustering": "task: clustering | query: ",
+    "Classification": "task: classification | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-query": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: "
+}
+# 1. Define distinct query vs document wrappers
+QUERY_PREFIX = AVAILABLE_TASK["query"]
+DOC_PREFIX = AVAILABLE_TASK["Retrieval-document"]
+
+# 2. Apply them separately (No overlapping text injected across documents!)
+v_query_t = get_embedding(QUERY_PREFIX + query_raw)
+v_rel_t = get_embedding(DOC_PREFIX + doc_relevant_raw)
+v_irrel_t = get_embedding(DOC_PREFIX + doc_irrelevant_raw)
+
+# 3. Calculate similarity
+sim_rel_t = cosine_similarity(v_query_t, v_rel_t)
+sim_irrel_t = cosine_similarity(v_query_t, v_irrel_t)
+
+print(f"Tuned Cosine Similarity (Relevant Doc):     {sim_rel_t:.4f}")
+print(f"Tuned Cosine Similarity (Irrelevant Doc):   {sim_irrel_t:.4f}")
+print(f"Tuned Margin Gap:                           {sim_rel_t - sim_irrel_t:.4f}")