update the codebase poc ver1

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/README.md

@@ -0,0 +1,78 @@
+# Semantic Chunking & Supabase Upload
+
+Implements Stage 3 (semantic chunking) and Stage 4 (EmbeddingGemma embed + Supabase upsert) from `knowledge/spec/ingestion/`.
+
+## Layout
+
+```
+implementation/ingestion/
+├── assemble.py           # Book-aware logical-unit assembly
+├── chunker.py            # Header pre-split + semantic breakpoint split
+├── config.py             # Paths, chunker defaults, DB mapping
+├── embedding.py          # EmbeddingGemma ONNX wrapper (use_gemma.py aligned)
+├── metadata_parser.py    # source_path filename parsing
+├── pipeline.py           # CLI orchestrator
+├── sqlite_store.py       # Local semantic_chunks staging in corpus_db
+├── supabase_uploader.py  # Upsert to knowledge.* Supabase schema
+└── requirements.txt
+```
+
+## Prerequisites
+
+1. Checkpoint databases in `knowledge/corpus_db/v1/` with `processed_chunks` populated.
+2. Supabase migrations applied (`implementation/supabase/migrations/`).
+3. Secrets at `PILOT_PROJECT/secrets/aws_secret/.env`:
+
+```bash
+SUPABASE_URL=https://<project-ref>.supabase.co
+SUPABASE_PUBLISHABLE_KEY=<publishable-key>
+```
+
+4. **Upload credentials** (choose one):
+
+| Method | Required setup |
+|--------|----------------|
+| PostgreSQL (recommended) | `SUPABASE_DB_URL` or `DATABASE_URL` from Supabase Dashboard → Database → Connection string |
+| Supabase REST | Expose `knowledge` schema in Dashboard → Project Settings → API → Exposed schemas, plus `SUPABASE_SERVICE_ROLE_KEY` |
+| Supabase CLI | Run `supabase link` under `implementation/supabase/` (uses `supabase db query --linked`) |
+
+`SUPABASE_PUBLISHABLE_KEY` alone is not sufficient for ingestion writes.
+
+4. Conda environment:
+
+```bash
+conda activate vkist_ultra
+pip install -r implementation/ingestion/requirements.txt
+```
+
+## Usage
+
+From `knowledge/implementation/`:
+
+```bash
+# Full pipeline for one book (chunk + embed + upload)
+python -m ingestion.pipeline --books mor
+
+# All books
+python -m ingestion.pipeline --books mor oho tny ado
+
+# Chunk only (writes semantic_chunks to local SQLite)
+python -m ingestion.pipeline --books mor --chunk-only
+
+# Upload pending chunks only
+python -m ingestion.pipeline --books mor --upload-only
+
+# Re-chunk existing logical units
+python -m ingestion.pipeline --books mor --force
+```
+
+## Chunker version
+
+Active version: `header+semantic_v1` (see `semantic_chunking_spec.md`).
+
+## References
+
+- `spec/ingestion/semantic_chunking_spec.md`
+- `spec/ingestion/corpus_profiles.md`
+- `spec/ingestion/schema.md`
+- `tests/use_gemma.py` — embedding instruction prefixes

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Knowledge ingestion pipeline: semantic chunking and Supabase upload."""

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/__main__.py

@@ -0,0 +1,6 @@
+"""Module entrypoint for `python -m ingestion.pipeline`."""
+
+from ingestion.pipeline import main
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/assemble.py

@@ -0,0 +1,373 @@
+"""Book-aware logical-unit assembly from processed_chunks rows."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from .config import BATCH_BOUNDARY_MARKER, GLUED_WORD_RE, MOR_TOC_PATH, IMAGE_MARKER_RE, PROTECTED_BLOCK_RE, TABLE_BLOCK_RE, STRUCTURAL_LINE_RE, HYPHENATION_BREAK_RE, SENTENCE_END_RE
+from .metadata_parser import SourceMetadata, infer_book_id, parse_source_path
+
+@dataclass
+class ProcessedChunkRow:
+    """One extraction checkpoint row."""
+
+    row_id: int
+    source_path: str
+    markdown_content: str
+    metadata: SourceMetadata
+
+
+@dataclass
+class LogicalUnit:
+    """An assembled text unit ready for header/semantic chunking."""
+
+    logical_unit_id: str
+    book_id: str
+    content: str
+    source_extraction_ids: list[int]
+    page_start: int
+    page_end: int
+    section_id: int | None = None
+    subsection_id: str | None = None
+    parent_title: str | None = None
+    assembly_metadata: dict = field(default_factory=dict)
+
+
+def load_mor_titles(toc_path: Path = MOR_TOC_PATH) -> dict[str, str]:
+    """Build subsection_id -> title lookup from MOR TOC JSON."""
+    with toc_path.open(encoding="utf-8") as handle:
+        payload = json.load(handle)
+
+    titles: dict[str, str] = {}
+    for section in payload.get("sections", []):
+        for subsection in section.get("subsections", []):
+            title = subsection.get("title", "")
+            match = re.match(r"^(\d+\.\d+)\b", title)
+            if match:
+                titles[match.group(1)] = title
+    return titles
+
+
+def _sort_key(row: ProcessedChunkRow) -> tuple:
+    meta = row.metadata
+    return (
+        meta.page_start,
+        meta.batch_index if meta.batch_index is not None else 0,
+        row.row_id,
+    )
+
+
+def _slugify_header(header: str) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", header.lower()).strip("_")
+    return slug[:80] or "untitled"
+
+
+def _logical_unit_anchor(start: int, header_title: str) -> str:
+    """Return a stable short suffix for repeated header titles in loose-tier books."""
+    digest = hashlib.sha256(f"{start}:{header_title}".encode("utf-8")).hexdigest()
+    return digest[:8]
+
+def _wrap_markdown_tables(text: str) -> str:
+    def replacer(match: re.Match[str]) -> str:
+        table = match.group(0).strip()
+        # Skip if already wrapped
+        if table.startswith("<table>"):
+            return table
+        return f"<table>\n{table}\n</table>"
+    return TABLE_BLOCK_RE.sub(replacer, text)
+
+def _is_prose_continuation(
+    previous_line: str,
+    next_line: str,
+    *,
+    across_blank: bool = False,
+) -> bool:
+    """Return True when next_line likely continues previous_line across a break."""
+    if STRUCTURAL_LINE_RE.match(next_line):
+        return False
+    if next_line[0].islower():
+        return True
+    if across_blank:
+        return False
+    return not SENTENCE_END_RE.search(previous_line.rstrip())
+
+def _normalize_newlines(text: str) -> str:
+    """Fix Docling/PDF soft wraps while preserving markdown structure."""
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    text = HYPHENATION_BREAK_RE.sub(r"\1\2", text)
+
+    lines = text.split("\n")
+    out: list[str] = []
+    index = 0
+    while index < len(lines):
+        line = lines[index].rstrip()
+        if not line:
+            out.append("")
+            index += 1
+            continue
+
+        if STRUCTURAL_LINE_RE.match(line):
+            out.append(line)
+            index += 1
+            continue
+
+        while index + 1 < len(lines):
+            blank_skip = 0
+            next_index = index + 1
+            if not lines[next_index].strip():
+                blank_skip = 1
+                next_index += 1
+            if next_index >= len(lines):
+                break
+
+            nxt = lines[next_index].strip()
+            if not nxt:
+                break
+            if blank_skip and not _is_prose_continuation(line, nxt, across_blank=True):
+                break
+            if STRUCTURAL_LINE_RE.match(nxt):
+                break
+            if not blank_skip and not _is_prose_continuation(line, nxt):
+                break
+
+            line = f"{line} {nxt}"
+            index = next_index
+            lines[index] = ""
+
+        out.append(re.sub(r" {2,}", " ", line))
+        index += 1
+
+    normalized = "\n".join(out)
+    normalized = re.sub(r"\n{3,}", "\n\n", normalized)
+    return normalized.strip()
+
+def _split_loose_book_stream(
+    rows: list[ProcessedChunkRow],
+) -> list[LogicalUnit]:
+    """Split a stitched loose-tier book stream into header-based logical units."""
+    if not rows:
+        return []
+
+    book_id = rows[0].metadata.book_id
+    stitched_parts: list[str] = []
+    extraction_ids: list[int] = []
+    for row in rows:
+        stitched_parts.append(row.markdown_content.strip())
+        extraction_ids.append(row.row_id)
+
+    book_stream = _clean_assembled_text(
+        BATCH_BOUNDARY_MARKER.join(stitched_parts).replace(BATCH_BOUNDARY_MARKER, "\n\n"),
+        book_id=book_id,
+    )
+    page_start = min(row.metadata.page_start for row in rows)
+    page_end = max(row.metadata.page_end for row in rows)
+
+    header_pattern = re.compile(r"^(#{1,2})\s+(.+)$", re.MULTILINE)
+    matches = list(header_pattern.finditer(book_stream))
+    if not matches:
+        return [
+            LogicalUnit(
+                logical_unit_id=f"{book_id}_book",
+                book_id=book_id,
+                content=book_stream.strip(),
+                source_extraction_ids=extraction_ids,
+                page_start=page_start,
+                page_end=page_end,
+                assembly_metadata={"strategy": "full_book_stream"},
+            )
+        ]
+
+    units: list[LogicalUnit] = []
+    for index, match in enumerate(matches):
+        start = match.start()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(book_stream)
+        section_text = book_stream[start:end].strip()
+        if not section_text:
+            continue
+
+        header_title = match.group(2).strip()
+        header_level = len(match.group(1))
+        unit_suffix = _slugify_header(header_title)
+        anchor = _logical_unit_anchor(start, header_title)
+        logical_unit_id = f"{book_id}_h{header_level}_{unit_suffix}_{anchor}"
+
+        units.append(
+            LogicalUnit(
+                logical_unit_id=logical_unit_id,
+                book_id=book_id,
+                content=section_text,
+                source_extraction_ids=extraction_ids,
+                page_start=page_start,
+                page_end=page_end,
+                parent_title=header_title,
+                assembly_metadata={"strategy": "header_boundary", "header_level": header_level},
+            )
+        )
+    return units
+
+def assemble_logical_units(
+    rows: list[tuple[int, str, str]],
+    book_id: str,
+    mor_titles: dict[str, str] | None = None,
+) -> list[LogicalUnit]:
+    """Assemble checkpoint rows into logical units according to corpus_profiles.md."""
+    parsed_rows = [
+        ProcessedChunkRow(
+            row_id=row_id,
+            source_path=source_path,
+            markdown_content=markdown_content,
+            metadata=parse_source_path(source_path, book_id=book_id),
+        )
+        for row_id, source_path, markdown_content in rows
+    ]
+    parsed_rows.sort(key=_sort_key)
+
+    if book_id in {"ado", "tny"}:
+        return _split_loose_book_stream(parsed_rows)
+
+    if book_id == "oho":
+        grouped: dict[int, list[ProcessedChunkRow]] = {}
+        for row in parsed_rows:
+            section_id = row.metadata.section_id
+            assert section_id is not None
+            grouped.setdefault(section_id, []).append(row)
+
+        units: list[LogicalUnit] = []
+        for section_id in sorted(grouped):
+            section_rows = grouped[section_id]
+            stitched = BATCH_BOUNDARY_MARKER.join(
+                row.markdown_content.strip() for row in section_rows
+            )
+            content = _clean_assembled_text(
+                stitched.replace(BATCH_BOUNDARY_MARKER, "\n\n"),
+                book_id=book_id,
+            )
+            units.append(
+                LogicalUnit(
+                    logical_unit_id=f"sec_{section_id}",
+                    book_id=book_id,
+                    content=content.strip(),
+                    source_extraction_ids=[row.row_id for row in section_rows],
+                    page_start=min(row.metadata.page_start for row in section_rows),
+                    page_end=max(row.metadata.page_end for row in section_rows),
+                    section_id=section_id,
+                    assembly_metadata={"strategy": "section_stitch", "batch_count": len(section_rows)},
+                )
+            )
+        return units
+
+    if book_id == "mor":
+        titles = mor_titles or load_mor_titles()
+        units = []
+        for row in parsed_rows:
+            subsection_id = row.metadata.subsection_id
+            assert subsection_id is not None
+            units.append(
+                LogicalUnit(
+                    logical_unit_id=f"sub_{subsection_id}",
+                    book_id=book_id,
+                    content=_clean_assembled_text(row.markdown_content, book_id=book_id),
+                    source_extraction_ids=[row.row_id],
+                    page_start=row.metadata.page_start,
+                    page_end=row.metadata.page_end,
+                    section_id=row.metadata.section_id,
+                    subsection_id=subsection_id,
+                    parent_title=titles.get(subsection_id),
+                    assembly_metadata={"strategy": "one_row_one_unit"},
+                )
+            )
+        return units
+
+    raise ValueError(f"Unsupported book_id for assembly: {book_id}")
+
+def _normalize_table_token_boundaries(text: str) -> str:
+      """Ensure block-level separation around wrapped tables."""
+      # </table> glued to following content (e.g. </table>## Juvenile SLE)
+      text = re.sub(r"</table>(?=\S)", "</table>\n\n", text)
+      # prose glued to opening tag (e.g. sentence<table>)
+      text = re.sub(r"(?<=\S)<table>", "\n\n<table>", text)
+      # optional: header glued before table (e.g. ## Title<table>)
+      text = re.sub(r"(#{1,6}\s[^\n]+)<table>", r"\1\n\n<table>", text)
+      return text
+
+def _normalize_header_lines(text: str) -> str:
+      """Collapse internal spacing in markdown headers."""
+      def _collapse_header(match: re.Match[str]) -> str:
+          prefix, title = match.group(1), match.group(2)
+          return f"{prefix}{' '.join(title.split())}"
+      return re.sub(
+          r"^(#{1,6}\s+)(.+)$",
+          _collapse_header,
+          text,
+          flags=re.MULTILINE,
+      )
+
+def _collapse_inline_whitespace(line: str) -> str:
+      # Handles spaces, tabs, nbsp, etc.
+      return " ".join(line.split())
+
+def _normalize_middle_dot_bullets(text: str) -> str:
+      # "plaques. · Lesions" → "plaques.\n\n- Lesions"  (optional list form)
+      text = re.sub(r"\s*·\s*", "\n- ", text)
+      # OR lighter touch — just normalize spacing:
+      # text = re.sub(r"\s*·\s*", " · ", text)
+      return text
+
+def _repair_glued_words(line: str) -> str:
+      def fix(m: re.Match[str]) -> str:
+          prefix, suffix = m.group(1), m.group(2)
+          # Avoid breaking real words: "mayo", "island", "androgen"
+          if suffix.lower() in {"o", "be", "occur", "also", "not"}:  # extend as needed
+              return f"{prefix} {suffix}"
+          return m.group(0)
+      return GLUED_WORD_RE.sub(fix, line)
+
+def _normalize_prose_spacing(text: str) -> str:
+      placeholders: dict[str, str] = {}
+      def _mask(match: re.Match[str]) -> str:
+          key = f"__PROTECTED_{len(placeholders)}__"
+          placeholders[key] = match.group(0)
+          return key
+      masked = PROTECTED_BLOCK_RE.sub(_mask, text)
+      lines = []
+      for line in masked.split("\n"):
+          if not line.strip():
+              lines.append("")
+              continue
+          if STRUCTURAL_LINE_RE.match(line):
+              lines.append(line)
+              continue
+          line = _collapse_inline_whitespace(line)
+          line = _repair_glued_words(line)  # optional
+          lines.append(line)
+      restored = "\n".join(lines)
+      for key, value in placeholders.items():
+          restored = restored.replace(key, value)
+      return _normalize_middle_dot_bullets(restored)  # optional
+
+def _clean_assembled_text(text: str, book_id: str | None = None) -> str:
+      cleaned = IMAGE_MARKER_RE.sub("", text)
+      cleaned = _normalize_newlines(cleaned)
+      cleaned = _wrap_markdown_tables(cleaned)
+      cleaned = _normalize_table_token_boundaries(cleaned)   # </table>## fix
+      cleaned = _normalize_prose_spacing(cleaned)            # NEW — intra-word spaces
+      cleaned = _normalize_header_lines(cleaned)
+      cleaned = re.sub(r"\n{3,}", "\n\n", cleaned)
+      return cleaned.strip()
+
+
+def rows_for_book(
+    all_rows: list[tuple[int, str, str]],
+    book_id: str,
+) -> list[tuple[int, str, str]]:
+    """Filter mixed checkpoint rows to a single book."""
+    filtered = []
+    for row in all_rows:
+        inferred = infer_book_id(row[1])
+        if inferred == book_id:
+            filtered.append(row)
+    return filtered

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/chunker.py

@@ -0,0 +1,399 @@
+"""Header and semantic chunking for retrieval-ready markdown units."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import uuid
+from dataclasses import dataclass
+from typing import Literal
+
+import numpy as np
+from langchain_text_splitters import MarkdownHeaderTextSplitter
+
+from .assemble import LogicalUnit, _wrap_markdown_tables
+from .config import (
+    CAPTION_LINE_RE,
+    CHUNKER_VERSION,
+    ChunkerConfig,
+    CODE_FENCE_RE,
+    DEFAULT_CHUNKER_CONFIG,
+    SENTENCE_SPLIT_RE,
+    TABLE_BLOCK_RE,
+    WRAPPED_TABLE_RE,
+)
+from .embedding import GemmaEmbedder, cosine_similarity
+
+logger = logging.getLogger(__name__)
+
+BlockType = Literal["prose", "table"]
+
+
+@dataclass(frozen=True)
+class ContentBlock:
+    """A prose or table segment extracted before chunking."""
+
+    block_type: BlockType
+    text: str
+    context: str | None = None
+
+
+@dataclass(frozen=True)
+class SemanticChunkRecord:
+    """One chunk ready for SQLite staging and Supabase upsert."""
+
+    chunk_uuid: str
+    logical_unit_id: str
+    chunk_index: int
+    content: str
+    token_count: int
+    book_id: str
+    section_id: int | None
+    subsection_id: str | None
+    parent_title: str | None
+    page_start: int | None
+    page_end: int | None
+    source_extraction_ids: list[int]
+    chunker_version: str
+    content_hash: str
+    embed_content: str | None = None
+
+
+def content_hash(content: str) -> str:
+    """Return SHA-256 hex digest for deduplication."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def embedding_text(record: SemanticChunkRecord) -> str:
+    """Return chunk body for retrieval embedding (title is passed separately to the embedder)."""
+    if record.embed_content:
+        return record.embed_content
+    if "<table>" in record.content:
+        return _table_embed_content(record.content, context=None)
+    return record.content
+
+
+def _headers_for_book(book_id: str) -> list[tuple[str, str]]:
+    if book_id in {"ado", "tny"}:
+        return [("#", "h1"), ("##", "h2"), ("###", "h3")]
+    return [("##", "h2"), ("###", "h3")]
+
+
+def _peel_trailing_caption(prose: str) -> tuple[str, str | None]:
+    """Move a trailing table/figure caption from prose onto the next table block."""
+    lines = prose.rstrip().split("\n")
+    if not lines:
+        return prose, None
+
+    last = lines[-1].strip()
+    if CAPTION_LINE_RE.match(last):
+        remaining = "\n".join(lines[:-1]).strip()
+        return remaining, last
+    return prose, None
+
+
+def _table_context_from_prose(prose: str) -> str | None:
+    """Return caption or last sentence from prose immediately before a table."""
+    prose = prose.strip()
+    if not prose:
+        return None
+
+    _, caption = _peel_trailing_caption(prose)
+    if caption:
+        return caption
+
+    sentences = _split_sentences(prose)
+    last_sentence = sentences[-1].strip() if sentences else ""
+    return last_sentence or None
+
+
+def _needs_table_wrap(text: str) -> bool:
+    """Return True when pipe markdown tables exist without assembly wrappers."""
+    if WRAPPED_TABLE_RE.search(text):
+        return False
+    return TABLE_BLOCK_RE.search(text) is not None
+
+
+def _prepare_text_for_extract(text: str) -> str:
+    """Normalize table markers only when assembly has not already wrapped them."""
+    stripped = text.strip()
+    if not stripped:
+        return ""
+    if _needs_table_wrap(stripped):
+        return _wrap_markdown_tables(stripped)
+    return stripped
+
+
+def extract_blocks(text: str) -> list[ContentBlock]:
+    """Split assembled markdown into alternating prose and table blocks."""
+    normalized = _prepare_text_for_extract(text)
+    if not normalized:
+        return []
+
+    if not WRAPPED_TABLE_RE.search(normalized):
+        return [ContentBlock(block_type="prose", text=normalized)]
+
+    blocks: list[ContentBlock] = []
+    position = 0
+
+    for match in WRAPPED_TABLE_RE.finditer(normalized):
+        prose_before = normalized[position : match.start()].strip()
+        caption: str | None = None
+        if prose_before:
+            prose_before, caption = _peel_trailing_caption(prose_before)
+            if prose_before:
+                blocks.append(ContentBlock(block_type="prose", text=prose_before))
+
+        table_text = match.group(0).strip()
+        if caption:
+            table_text = f"{caption}\n\n{table_text}"
+
+        context = caption or _table_context_from_prose(prose_before)
+        blocks.append(
+            ContentBlock(
+                block_type="table",
+                text=table_text,
+                context=context,
+            )
+        )
+        position = match.end()
+
+    trailing = normalized[position:].strip()
+    if trailing:
+        blocks.append(ContentBlock(block_type="prose", text=trailing))
+
+    return blocks
+
+
+def _split_sentences(text: str) -> list[str]:
+    protected = text
+    placeholders: dict[str, str] = {}
+
+    for index, match in enumerate(CODE_FENCE_RE.finditer(text)):
+        key = f"__CODE_BLOCK_{index}__"
+        placeholders[key] = match.group(0)
+        protected = protected.replace(match.group(0), key, 1)
+
+    for index, match in enumerate(TABLE_BLOCK_RE.finditer(protected)):
+        key = f"__TABLE_BLOCK_{index}__"
+        placeholders[key] = match.group(0)
+        protected = protected.replace(match.group(0), key, 1)
+
+    parts = SENTENCE_SPLIT_RE.split(protected.strip())
+    restored = []
+    for part in parts:
+        chunk = part.strip()
+        if not chunk:
+            continue
+        for key, value in placeholders.items():
+            chunk = chunk.replace(key, value)
+        restored.append(chunk)
+    return restored or [text.strip()]
+
+
+def _header_split_text(text: str, book_id: str) -> list[tuple[str, dict]]:
+    splitter = MarkdownHeaderTextSplitter(
+        headers_to_split_on=_headers_for_book(book_id),
+        strip_headers=False,
+    )
+    docs = splitter.split_text(text)
+    sections: list[tuple[str, dict]] = []
+    for doc in docs:
+        section_text = doc.page_content.strip()
+        if not section_text:
+            continue
+        sections.append((section_text, dict(doc.metadata)))
+    if not sections:
+        sections.append((text.strip(), {}))
+    return sections
+
+
+def _resolve_parent_title(unit: LogicalUnit, header_meta: dict) -> str | None:
+    if unit.parent_title:
+        return unit.parent_title
+    for key in ("h3", "h2", "h1"):
+        if header_meta.get(key):
+            return header_meta[key]
+    return None
+
+
+def _semantic_split_text(
+    text: str,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[str]:
+    sentences = _split_sentences(text)
+    if len(sentences) <= 1:
+        return [text.strip()]
+
+    embeddings = [embedder.embed_clustering(sentence) for sentence in sentences]
+    distances = [
+        1.0 - cosine_similarity(embeddings[index], embeddings[index + 1])
+        for index in range(len(embeddings) - 1)
+    ]
+
+    if not distances:
+        return [text.strip()]
+
+    threshold = float(np.percentile(distances, config.breakpoint_threshold_amount))
+    breakpoints = [0]
+    for index, distance in enumerate(distances):
+        if distance >= threshold:
+            breakpoints.append(index + 1)
+    breakpoints.append(len(sentences))
+
+    chunks: list[str] = []
+    for start, end in zip(breakpoints[:-1], breakpoints[1:]):
+        chunk_text = " ".join(sentences[start:end]).strip()
+        if chunk_text:
+            chunks.append(chunk_text)
+    return chunks or [text.strip()]
+
+
+def _split_oversized_piece(
+    text: str,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[str]:
+    token_count = embedder.count_tokens(text)
+    hard_cap = int(config.max_tokens * config.hard_cap_multiplier)
+    if token_count <= config.max_tokens:
+        return [text]
+
+    if token_count <= hard_cap and TABLE_BLOCK_RE.search(text):
+        return [text]
+
+    return _semantic_split_text(text, embedder, config)
+
+
+def _chunk_prose_block(
+    prose_text: str,
+    unit: LogicalUnit,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[tuple[str, str | None, BlockType, str | None]]:
+    """Header- and semantic-split one prose block; never merges with tables."""
+    pieces: list[tuple[str, str | None, BlockType, str | None]] = []
+    header_sections = _header_split_text(prose_text, unit.book_id)
+
+    for section_text, header_meta in header_sections:
+        parent_title = _resolve_parent_title(unit, header_meta)
+        if unit.book_id == "mor" and embedder.count_tokens(section_text) <= config.max_tokens:
+            pieces.append((section_text, parent_title, "prose", None))
+            continue
+
+        for piece in _split_oversized_piece(section_text, embedder, config):
+            pieces.append((piece, parent_title, "prose", None))
+
+    return pieces
+
+
+def _table_embed_content(
+    table_content: str,
+    *,
+    context: str | None,
+) -> str:
+    """Build table chunk body for retrieval embedding (title uses the embedder prefix)."""
+    if context and context not in table_content:
+        return f"{context.strip()}\n\n{table_content.strip()}"
+    return table_content.strip()
+
+
+def _merge_tiny_prose_pieces(
+    pieces: list[tuple[str, str | None, BlockType, str | None]],
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig,
+) -> list[tuple[str, str | None, BlockType, str | None]]:
+    """Merge undersized prose chunks only; tables stay isolated."""
+    if not pieces:
+        return []
+
+    merged: list[tuple[str, str | None, BlockType, str | None]] = []
+    carry_text: str | None = None
+    carry_title: str | None = None
+
+    for text, title, block_type, context in pieces:
+        if block_type == "table":
+            if carry_text is not None:
+                merged.append((carry_text, carry_title, "prose", None))
+                carry_text, carry_title = None, None
+            merged.append((text, title, "table", context))
+            continue
+
+        if carry_text is None:
+            carry_text, carry_title = text, title
+            continue
+
+        carry_tokens = embedder.count_tokens(carry_text)
+        piece_tokens = embedder.count_tokens(text)
+        max_merge_size = int(config.max_tokens * 1.25)
+        if carry_tokens < config.min_tokens and carry_tokens + piece_tokens <= max_merge_size:
+            carry_text = f"{carry_text}\n\n{text}".strip()
+            if not carry_title:
+                carry_title = title
+        else:
+            merged.append((carry_text, carry_title, "prose", None))
+            carry_text, carry_title = text, title
+
+    if carry_text is not None:
+        merged.append((carry_text, carry_title, "prose", None))
+
+    return merged
+
+
+def chunk_logical_unit(
+    unit: LogicalUnit,
+    embedder: GemmaEmbedder,
+    config: ChunkerConfig = DEFAULT_CHUNKER_CONFIG,
+    chunker_version: str = CHUNKER_VERSION,
+) -> list[SemanticChunkRecord]:
+    """Chunk one logical unit using prose/table split, header pre-split, and semantic refinement."""
+    content_blocks = extract_blocks(unit.content)
+    raw_pieces: list[tuple[str, str | None, BlockType, str | None]] = []
+
+    for block in content_blocks:
+        if block.block_type == "table":
+            raw_pieces.append((block.text, unit.parent_title, "table", block.context))
+            continue
+        raw_pieces.extend(_chunk_prose_block(block.text, unit, embedder, config))
+
+    merged_pieces = _merge_tiny_prose_pieces(raw_pieces, embedder, config)
+
+    records: list[SemanticChunkRecord] = []
+    seen_hashes: set[str] = set()
+
+    for index, (text, parent_title, block_type, context) in enumerate(merged_pieces):
+        normalized = text.strip()
+        if not normalized:
+            continue
+        digest = content_hash(normalized)
+        if digest in seen_hashes:
+            logger.debug("Skipping duplicate chunk content in %s", unit.logical_unit_id)
+            continue
+        seen_hashes.add(digest)
+
+        resolved_title = parent_title or unit.parent_title
+        embed_content = None
+        if block_type == "table":
+            embed_content = _table_embed_content(normalized, context=context)
+
+        records.append(
+            SemanticChunkRecord(
+                chunk_uuid=str(uuid.uuid4()),
+                logical_unit_id=unit.logical_unit_id,
+                chunk_index=index,
+                content=normalized,
+                token_count=embedder.count_tokens(normalized),
+                book_id=unit.book_id,
+                section_id=unit.section_id,
+                subsection_id=unit.subsection_id,
+                parent_title=resolved_title,
+                page_start=unit.page_start,
+                page_end=unit.page_end,
+                source_extraction_ids=list(unit.source_extraction_ids),
+                chunker_version=chunker_version,
+                content_hash=digest,
+                embed_content=embed_content,
+            )
+        )
+    return records

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/config.py

@@ -0,0 +1,100 @@
+"""Configuration for the knowledge ingestion pipeline."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+import re
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parents[2]
+IMPLEMENTATION_ROOT = Path(__file__).resolve().parents[1]
+CORPUS_DB_DIR = KNOWLEDGE_ROOT / "corpus_db" / "v1"
+CORPUS_ROOT = KNOWLEDGE_ROOT / "corpus"
+MODEL_DIR = IMPLEMENTATION_ROOT / "model" / "gemma_final"
+MOR_TOC_PATH = CORPUS_ROOT / "pdf" / "mor" / "toc_structure.json"
+IMAGE_MARKER_RE = re.compile(r"^\s*<!--\s*image\s*-->\s*$", re.MULTILINE | re.IGNORECASE)
+DEFAULT_ENV_PATH = (
+    Path(__file__).resolve().parents[6]
+    / "secrets"
+    / "aws_secret"
+    / ".env"
+)
+TABLE_BLOCK_RE = re.compile(
+    r"(?:^\|.+\|\s*\n(?:^\|[-:\s|]+\|\s*\n)?(?:^\|.+\|\s*\n?)+)",
+    re.MULTILINE,
+)
+
+WRAPPED_TABLE_RE = re.compile(r"<table>[\s\S]*?</table>", re.MULTILINE)
+
+CAPTION_LINE_RE = re.compile(
+    r"^(?:Table|Figure|Fig\.)\s+[\dIVXivx]+",
+    re.IGNORECASE,
+)
+
+CODE_FENCE_RE = re.compile(r"```[\s\S]*?```", re.MULTILINE)
+
+SENTENCE_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9\"(])")
+
+CHUNKER_VERSION = "header+semantic_v2"
+EMBEDDING_MODEL_NAME = "EmbeddingGemma"
+EMBEDDING_MODEL_VERSION = "1"
+EMBEDDING_DIMENSIONS = 768
+EDITION_LABEL = "v1"
+
+BATCH_BOUNDARY_MARKER = "\n\n<!-- batch-boundary -->\n\n"
+
+BOOK_DB_FILES: dict[str, str] = {
+    "ado": "ado_ingestion_corpus.db",
+    "tny": "tny_ingestion_corpus.db",
+    "oho": "oho_ingestion_corpus.db",
+    "mor": "mor_ingestion_corpus.db",
+}
+
+ADO_LEGACY_FILENAME_RE = re.compile(
+      r"^chunk_(?P<batch_index>\d+)-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+  )
+
+PROTECTED_BLOCK_RE = re.compile(
+    r"(<table>[\s\S]*?</table>|```[\s\S]*?```|"
+    r"(?:^\|.+\|\s*\n(?:^\|[-:\s|]+\|\s*\n)?(?:^\|.+\|\s*\n?)+))",
+    re.MULTILINE,
+)
+
+GLUED_WORD_RE = re.compile(
+      r"\b(may|can|are|is|and|the)([a-z]{3,})\b",
+      re.IGNORECASE,
+  )
+
+CONSTRAINT_TIERS: dict[str, str] = {
+    "ado": "loose",
+    "tny": "loose",
+    "oho": "section",
+    "mor": "subsection",
+}
+
+STRUCTURAL_LINE_RE = re.compile(r"^(#{1,6}\s|[-*+]\s|\d+\.\s|```|\|)")
+HYPHENATION_BREAK_RE = re.compile(r"(\w)-\n(\w)")
+SENTENCE_END_RE = re.compile(r'[.!?:]["\']?\s*$')
+
+
+@dataclass(frozen=True)
+class ChunkerConfig:
+    """Semantic chunking parameters from semantic_chunking_spec.md."""
+
+    max_tokens: int = 800
+    min_tokens: int = 120
+    overlap_tokens: int = 64
+    breakpoint_threshold_type: str = "percentile"
+    breakpoint_threshold_amount: float = 95.0
+    hard_cap_multiplier: float = 1.5
+
+
+DEFAULT_CHUNKER_CONFIG = ChunkerConfig()
+
+
+def resolve_corpus_db_path(book_id: str) -> Path:
+    """Return the SQLite checkpoint database path for a textbook."""
+    filename = BOOK_DB_FILES.get(book_id)
+    if filename is None:
+        raise ValueError(f"Unknown book_id: {book_id}")
+    return CORPUS_DB_DIR / filename

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/embedding.py

@@ -0,0 +1,149 @@
+"""EmbeddingGemma ONNX embedding wrapper aligned with use_gemma.py."""
+
+from __future__ import annotations
+
+import logging
+from enum import Enum
+from functools import lru_cache
+from pathlib import Path
+
+import numpy as np
+import torch
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+from transformers import AutoTokenizer
+
+from .config import EMBEDDING_DIMENSIONS, MODEL_DIR
+
+logger = logging.getLogger(__name__)
+
+
+class EmbedTask(str, Enum):
+    """EmbeddingGemma instruction-routing tasks used by this pipeline."""
+
+    CLUSTERING = "Clustering"
+    RETRIEVAL_DOCUMENT = "Retrieval-document"
+    RETRIEVAL_QUERY = "Retrieval-query"
+
+
+# Full task map from EmbeddingGemma / MTEB; pipeline uses EmbedTask subset only.
+TASK_PREFIXES: dict[str, str] = {
+    "query": "task: search result | query: ",
+    "document": "title: none | text: ",
+    "BitextMining": "task: search result | query: ",
+    "Clustering": "task: clustering | query: ",
+    "Classification": "task: classification | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-query": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: ",
+}
+
+# Backward-compatible aliases.
+AVAILABLE_TASK = TASK_PREFIXES
+DOC_PREFIX = TASK_PREFIXES["Retrieval-document"]
+QUERY_PREFIX = TASK_PREFIXES["Retrieval-query"]
+
+
+def format_embed_input(
+    text: str,
+    task: EmbedTask,
+    *,
+    title: str | None = None,
+) -> str:
+    """Apply the EmbeddingGemma instruction prefix for a pipeline role."""
+    if task == EmbedTask.RETRIEVAL_DOCUMENT:
+        title_value = title.strip() if title and title.strip() else "none"
+        return f"title: {title_value} | text: {text}"
+    return TASK_PREFIXES[task.value] + text
+
+
+class GemmaEmbedder:
+    """Local EmbeddingGemma feature extractor with task-routed instruction prefixes."""
+
+    def __init__(
+        self,
+        model_dir: Path = MODEL_DIR,
+        target_dim: int = EMBEDDING_DIMENSIONS,
+        provider: str = "CPUExecutionProvider",
+    ) -> None:
+        """Load tokenizer and ONNX model from the local model directory."""
+        if not hasattr(torch, "int4"):
+            torch.int4 = torch.int8
+
+        self.target_dim = target_dim
+        self.tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True)
+        self.model = ORTModelForFeatureExtraction.from_pretrained(
+            model_dir,
+            file_name="model_fp16.onnx",
+            local_files_only=True,
+            provider=provider,
+            use_io_binding=False,
+        )
+        logger.info("Loaded EmbeddingGemma from %s", model_dir)
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens using the embedding model tokenizer."""
+        return len(
+            self.tokenizer.encode(
+                text,
+                add_special_tokens=True,
+                truncation=False,
+            )
+        )
+
+    def embed(
+        self,
+        text: str,
+        task: EmbedTask,
+        *,
+        title: str | None = None,
+    ) -> np.ndarray:
+        """Embed text with the instruction prefix for the given pipeline task."""
+        return self._embed(format_embed_input(text, task, title=title))
+
+    def embed_document(self, text: str, *, title: str | None = None) -> np.ndarray:
+        """Embed a retrieval corpus chunk (asymmetric document side)."""
+        return self.embed(text, EmbedTask.RETRIEVAL_DOCUMENT, title=title)
+
+    def embed_query(self, text: str) -> np.ndarray:
+        """Embed a retrieval search query (asymmetric query side)."""
+        return self.embed(text, EmbedTask.RETRIEVAL_QUERY)
+
+    def embed_clustering(self, text: str) -> np.ndarray:
+        """Embed text for semantic breakpoint / clustering during chunking."""
+        return self.embed(text, EmbedTask.CLUSTERING)
+
+    def _embed(self, text: str) -> np.ndarray:
+        inputs = self.tokenizer(
+            text,
+            max_length=512,
+            padding=True,
+            truncation=True,
+            return_tensors="pt",
+        )
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+        raw_states = outputs.last_hidden_state.numpy()
+        mean_pooled = np.mean(raw_states, axis=1)[0]
+        vector = mean_pooled[: self.target_dim]
+        norm = np.linalg.norm(vector)
+        if norm == 0:
+            return vector
+        return vector / norm
+
+
+@lru_cache(maxsize=1)
+def get_embedder() -> GemmaEmbedder:
+    """Return a process-wide singleton embedder instance."""
+    return GemmaEmbedder()
+
+
+def cosine_similarity(vector_a: np.ndarray, vector_b: np.ndarray) -> float:
+    """Compute cosine similarity between two vectors."""
+    denom = np.linalg.norm(vector_a) * np.linalg.norm(vector_b)
+    if denom == 0:
+        return 0.0
+    return float(np.dot(vector_a, vector_b) / denom)

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/metadata_parser.py

@@ -0,0 +1,122 @@
+"""Parse source_path filenames into structured ingestion metadata."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from .config import ADO_LEGACY_FILENAME_RE
+
+
+BATCH_FILENAME_RE = re.compile(
+    r"^batch-(?P<batch_index>\d+)_chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+)
+OHO_FILENAME_RE = re.compile(
+    r"^sec_(?P<section_id>\d+)_batch-(?P<batch_index>\d+)_"
+    r"chunk-(?P<page_start>\d+)-(?P<page_end>\d+)\.pdf$"
+)
+MOR_FILENAME_RE = re.compile(
+    r"^sub_(?P<subsection_id>\d+\.\d+)_chunk-(?P<page_start>\d+)-"
+    r"(?P<page_end>\d+)\.pdf$"
+)
+
+
+@dataclass(frozen=True)
+class SourceMetadata:
+    """Structured metadata extracted from a processed_chunks source_path."""
+
+    book_id: str
+    page_start: int
+    page_end: int
+    section_id: int | None
+    subsection_id: str | None
+    batch_index: int | None
+    source_filename: str
+
+
+def infer_book_id(source_path: str) -> str:
+    """Infer book_id from the source_path string."""
+    normalized = source_path.replace("\\", "/").lower()
+    for book_id in ("mor", "oho", "tny", "ado"):
+        if f"/{book_id}/" in normalized or f"/pdf/{book_id}/" in normalized:
+            return book_id
+    raise ValueError(f"Cannot infer book_id from source_path: {source_path}")
+
+
+def parse_source_path(source_path: str, book_id: str | None = None) -> SourceMetadata:
+    """Parse a checkpoint row source_path into structured metadata."""
+    resolved_book_id = book_id or infer_book_id(source_path)
+    filename = Path(source_path).name
+
+    if resolved_book_id == "oho":
+        match = OHO_FILENAME_RE.match(filename)
+        if not match:
+            raise ValueError(f"OHO filename does not match expected pattern: {filename}")
+        groups = match.groupdict()
+        return SourceMetadata(
+            book_id=resolved_book_id,
+            page_start=int(groups["page_start"]),
+            page_end=int(groups["page_end"]),
+            section_id=int(groups["section_id"]),
+            subsection_id=None,
+            batch_index=int(groups["batch_index"]),
+            source_filename=filename,
+        )
+
+    if resolved_book_id == "mor":
+        match = MOR_FILENAME_RE.match(filename)
+        if not match:
+            raise ValueError(f"MOR filename does not match expected pattern: {filename}")
+        groups = match.groupdict()
+        subsection_id = groups["subsection_id"]
+        return SourceMetadata(
+            book_id=resolved_book_id,
+            page_start=int(groups["page_start"]),
+            page_end=int(groups["page_end"]),
+            section_id=int(subsection_id.split(".")[0]),
+            subsection_id=subsection_id,
+            batch_index=None,
+            source_filename=filename,
+        )
+
+    if resolved_book_id == "ado":
+          legacy_match = ADO_LEGACY_FILENAME_RE.match(filename)
+          if legacy_match:
+              groups = legacy_match.groupdict()
+              return SourceMetadata(
+                  book_id=resolved_book_id,
+                  page_start=int(groups["page_start"]),
+                  page_end=int(groups["page_end"]),
+                  section_id=None,
+                  subsection_id=None,
+                  batch_index=int(groups["batch_index"]),
+                  source_filename=filename,
+              )
+          
+    match = BATCH_FILENAME_RE.match(filename)
+    if not match:
+        raise ValueError(
+            f"{resolved_book_id.upper()} filename does not match expected pattern: {filename}"
+        )
+    groups = match.groupdict()
+    return SourceMetadata(
+        book_id=resolved_book_id,
+        page_start=int(groups["page_start"]),
+        page_end=int(groups["page_end"]),
+        section_id=None,
+        subsection_id=None,
+        batch_index=int(groups["batch_index"]),
+        source_filename=filename,
+    )
+
+
+def logical_unit_key(meta: SourceMetadata) -> str:
+    """Return a stable assembly key for grouping checkpoint rows."""
+    if meta.book_id == "mor":
+        return f"sub_{meta.subsection_id}"
+    if meta.book_id == "oho":
+        return f"sec_{meta.section_id}"
+    raise NotImplementedError(
+        "Loose-tier books assign logical_unit_id after book_stream section detection."
+    )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pg_psycopg.py

@@ -0,0 +1,193 @@
+"""psycopg2-backed PostgreSQL upload helpers."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from typing import Generator
+
+import numpy as np
+
+from .chunker import SemanticChunkRecord
+from .config import EDITION_LABEL, EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_VERSION
+from .pg_uploader import PostgresContext
+
+
+@contextmanager
+def postgres_connection(dsn: str) -> Generator:
+    import psycopg2
+    from pgvector.psycopg2 import register_vector
+
+    connection = psycopg2.connect(dsn)
+    register_vector(connection)
+    try:
+        yield connection
+        connection.commit()
+    except Exception:
+        connection.rollback()
+        raise
+    finally:
+        connection.close()
+
+
+def build_context_with_connection(dsn: str, book_ids: list[str]) -> tuple[dict[str, str], str]:
+    edition_ids: dict[str, str] = {}
+    model_id: str | None = None
+
+    with postgres_connection(dsn) as conn:
+        with conn.cursor() as cur:
+            for book_id in book_ids:
+                cur.execute(
+                    """
+                    SELECT ce.edition_id
+                    FROM knowledge.corpus_edition ce
+                    INNER JOIN knowledge.corpus_source cs
+                        ON cs.corpus_source_id = ce.corpus_source_id
+                    WHERE cs.book_id = %s AND ce.status = 'active'
+                    LIMIT 1
+                    """,
+                    (book_id,),
+                )
+                row = cur.fetchone()
+                if row:
+                    edition_ids[book_id] = str(row[0])
+                    continue
+
+                cur.execute(
+                    "SELECT corpus_source_id FROM knowledge.corpus_source WHERE book_id = %s",
+                    (book_id,),
+                )
+                source_row = cur.fetchone()
+                if not source_row:
+                    raise RuntimeError(f"Missing corpus_source seed row for book_id={book_id}")
+                cur.execute(
+                    """
+                    INSERT INTO knowledge.corpus_edition (
+                        corpus_source_id, edition_label, status
+                    ) VALUES (%s, %s, 'active')
+                    RETURNING edition_id
+                    """,
+                    (source_row[0], EDITION_LABEL),
+                )
+                edition_ids[book_id] = str(cur.fetchone()[0])
+
+            cur.execute(
+                """
+                SELECT model_id FROM knowledge.embedding_model
+                WHERE model_name = %s AND model_version = %s
+                LIMIT 1
+                """,
+                (EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_VERSION),
+            )
+            model_row = cur.fetchone()
+            if not model_row:
+                raise RuntimeError("Missing embedding_model seed row for EmbeddingGemma")
+            model_id = str(model_row[0])
+
+    return edition_ids, model_id
+
+
+def upload_with_connection(
+    dsn: str,
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None,
+) -> None:
+    edition_id = context.edition_ids[record.book_id]
+    with postgres_connection(dsn) as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO knowledge.logical_unit (
+                    edition_id, unit_key, parent_title, page_start, page_end
+                ) VALUES (%s, %s, %s, %s, %s)
+                ON CONFLICT (edition_id, unit_key) DO UPDATE SET
+                    parent_title = EXCLUDED.parent_title,
+                    page_start = EXCLUDED.page_start,
+                    page_end = EXCLUDED.page_end
+                RETURNING logical_unit_id
+                """,
+                (
+                    edition_id,
+                    record.logical_unit_id,
+                    record.parent_title,
+                    record.page_start,
+                    record.page_end,
+                ),
+            )
+            logical_unit_uuid = cur.fetchone()[0]
+
+            cur.execute(
+                """
+                INSERT INTO knowledge.semantic_chunk (
+                    chunk_id, logical_unit_id, edition_id, chunker_version,
+                    chunk_index, content, token_count, content_hash,
+                    section_id, subsection_id, parent_title, page_start, page_end,
+                    is_active
+                ) VALUES (
+                    %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s,
+                    %s, %s, %s, %s, %s, true
+                )
+                ON CONFLICT (logical_unit_id, chunk_index, chunker_version) DO UPDATE SET
+                    content = EXCLUDED.content,
+                    token_count = EXCLUDED.token_count,
+                    content_hash = EXCLUDED.content_hash,
+                    section_id = EXCLUDED.section_id,
+                    subsection_id = EXCLUDED.subsection_id,
+                    parent_title = EXCLUDED.parent_title,
+                    page_start = EXCLUDED.page_start,
+                    page_end = EXCLUDED.page_end,
+                    is_active = true
+                RETURNING chunk_id
+                """,
+                (
+                    record.chunk_uuid,
+                    logical_unit_uuid,
+                    edition_id,
+                    record.chunker_version,
+                    record.chunk_index,
+                    record.content,
+                    record.token_count,
+                    record.content_hash,
+                    record.section_id,
+                    record.subsection_id,
+                    record.parent_title,
+                    record.page_start,
+                    record.page_end,
+                ),
+            )
+            chunk_id = cur.fetchone()[0]
+
+            for extraction_id in record.source_extraction_ids:
+                cur.execute(
+                    """
+                    INSERT INTO knowledge.chunk_provenance (
+                        chunk_id, checkpoint_db, processed_chunk_id, source_path
+                    ) VALUES (%s::uuid, %s, %s, %s)
+                    ON CONFLICT (chunk_id, checkpoint_db, processed_chunk_id) DO UPDATE SET
+                        source_path = EXCLUDED.source_path
+                    """,
+                    (
+                        chunk_id,
+                        checkpoint_db,
+                        extraction_id,
+                        (source_paths or {}).get(
+                            extraction_id,
+                            f"processed_chunks:{extraction_id}",
+                        ),
+                    ),
+                )
+
+            cur.execute(
+                """
+                INSERT INTO knowledge.semantic_chunk_embedding (
+                    chunk_id, model_id, edition_id, embedding, embedding_status, embedded_at
+                ) VALUES (%s::uuid, %s::uuid, %s::uuid, %s, 'indexed', now())
+                ON CONFLICT (chunk_id, model_id) DO UPDATE SET
+                    embedding = EXCLUDED.embedding,
+                    embedding_status = 'indexed',
+                    embedded_at = now()
+                """,
+                (chunk_id, context.model_id, edition_id, vector.tolist()),
+            )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pg_uploader.py

@@ -0,0 +1,288 @@
+"""Direct PostgreSQL uploader for the knowledge schema."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from .chunker import SemanticChunkRecord
+from .config import (
+    EDITION_LABEL,
+    EMBEDDING_DIMENSIONS,
+    EMBEDDING_MODEL_NAME,
+    EMBEDDING_MODEL_VERSION,
+)
+
+logger = logging.getLogger(__name__)
+
+SUPABASE_DIR = Path(__file__).resolve().parents[1] / "supabase"
+
+
+@dataclass(frozen=True)
+class PostgresContext:
+    """Resolved identifiers for PostgreSQL upserts."""
+
+    edition_ids: dict[str, str]
+    model_id: str
+    mode: str
+    dsn: str | None = None
+
+
+def resolve_database_dsn() -> str:
+    """Resolve a PostgreSQL connection string from environment variables."""
+    for key in ("SUPABASE_DB_URL", "DATABASE_URL", "SUPABASE_DATABASE_URL"):
+        value = os.getenv(key)
+        if value:
+            return value
+    raise RuntimeError(
+        "No PostgreSQL DSN found. Set SUPABASE_DB_URL/DATABASE_URL or use "
+        "Supabase CLI linked-project mode."
+    )
+
+
+def _supabase_cli_available() -> bool:
+    project_ref = SUPABASE_DIR / ".temp" / "project-ref"
+    try:
+        subprocess.run(
+            ["supabase", "--version"],
+            capture_output=True,
+            check=True,
+            text=True,
+        )
+    except (FileNotFoundError, subprocess.CalledProcessError):
+        return False
+    return project_ref.exists()
+
+
+def _run_sql_via_cli(sql: str) -> dict[str, Any]:
+    """Execute SQL against the linked remote Supabase database."""
+    result = subprocess.run(
+        ["supabase", "db", "query", "--linked", sql],
+        cwd=SUPABASE_DIR,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    if not result.stdout.strip():
+        return {}
+    return json.loads(result.stdout)
+
+
+def _sql_literal(value: str | None) -> str:
+    if value is None:
+        return "NULL"
+    return "'" + value.replace("'", "''") + "'"
+
+
+def _query_rows(sql: str) -> list[dict[str, Any]]:
+    payload = _run_sql_via_cli(sql)
+    return payload.get("rows") or []
+
+
+def _query_scalar(sql: str) -> str | None:
+    rows = _query_rows(sql)
+    if not rows:
+        return None
+    return str(next(iter(rows[0].values())))
+
+
+def build_postgres_context(
+    book_ids: list[str],
+    dsn: str | None = None,
+) -> PostgresContext:
+    """Resolve edition and embedding model ids for upload."""
+    use_cli = dsn is None and _supabase_cli_available()
+    if not use_cli:
+        resolved_dsn = dsn or resolve_database_dsn()
+        return _build_context_with_psycopg(resolved_dsn, book_ids)
+
+    edition_ids: dict[str, str] = {}
+    for book_id in book_ids:
+        edition_id = _query_scalar(
+            f"""
+            SELECT ce.edition_id::text
+            FROM knowledge.corpus_edition ce
+            INNER JOIN knowledge.corpus_source cs
+                ON cs.corpus_source_id = ce.corpus_source_id
+            WHERE cs.book_id = {_sql_literal(book_id)} AND ce.status = 'active'
+            LIMIT 1
+            """
+        )
+        if not edition_id:
+            corpus_source_id = _query_scalar(
+                "SELECT corpus_source_id::text "
+                f"FROM knowledge.corpus_source WHERE book_id = {_sql_literal(book_id)}"
+            )
+            if not corpus_source_id:
+                raise RuntimeError(f"Missing corpus_source seed row for book_id={book_id}")
+            edition_id = _query_scalar(
+                f"""
+                INSERT INTO knowledge.corpus_edition (
+                    corpus_source_id, edition_label, status
+                ) VALUES ('{corpus_source_id}', {_sql_literal(EDITION_LABEL)}, 'active')
+                RETURNING edition_id::text
+                """
+            )
+        if not edition_id:
+            raise RuntimeError(f"Unable to resolve active edition for book_id={book_id}")
+        edition_ids[book_id] = edition_id
+
+    model_id = _query_scalar(
+        f"""
+        SELECT model_id::text FROM knowledge.embedding_model
+        WHERE model_name = {_sql_literal(EMBEDDING_MODEL_NAME)}
+          AND model_version = {_sql_literal(EMBEDDING_MODEL_VERSION)}
+        LIMIT 1
+        """
+    )
+    if not model_id:
+        raise RuntimeError("Missing embedding_model seed row for EmbeddingGemma")
+    return PostgresContext(edition_ids=edition_ids, model_id=model_id, mode="cli")
+
+
+def _build_context_with_psycopg(dsn: str, book_ids: list[str]) -> PostgresContext:
+    from .pg_psycopg import build_context_with_connection
+
+    edition_ids, model_id = build_context_with_connection(dsn, book_ids)
+    return PostgresContext(
+        edition_ids=edition_ids,
+        model_id=model_id,
+        mode="psycopg",
+        dsn=dsn,
+    )
+
+
+def upload_semantic_chunk_pg(
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    """Upsert one chunk and embedding to Supabase PostgreSQL."""
+    if vector.shape[0] != EMBEDDING_DIMENSIONS:
+        raise ValueError(
+            f"Expected {EMBEDDING_DIMENSIONS}-d embedding, got {vector.shape[0]}"
+        )
+
+    if context.mode == "cli":
+        _upload_via_cli(context, record, vector, checkpoint_db, source_paths)
+        return
+
+    from .pg_psycopg import upload_with_connection
+
+    if not context.dsn:
+        raise RuntimeError("Missing PostgreSQL DSN for psycopg upload mode.")
+    upload_with_connection(
+        context.dsn,
+        context,
+        record,
+        vector,
+        checkpoint_db,
+        source_paths,
+    )
+
+
+def _upload_via_cli(
+    context: PostgresContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None,
+) -> None:
+    edition_id = context.edition_ids[record.book_id]
+    vector_literal = json.dumps([float(v) for v in vector.tolist()])
+    provenance_values = []
+    for extraction_id in record.source_extraction_ids:
+        source_path = (source_paths or {}).get(
+            extraction_id,
+            f"processed_chunks:{extraction_id}",
+        )
+        provenance_values.append(
+            f"({_sql_literal(checkpoint_db)}, {extraction_id}, {_sql_literal(source_path)})"
+        )
+
+    sql = f"""
+    WITH upsert_unit AS (
+        INSERT INTO knowledge.logical_unit (
+            edition_id, unit_key, parent_title, page_start, page_end
+        ) VALUES (
+            '{edition_id}'::uuid,
+            {_sql_literal(record.logical_unit_id)},
+            {_sql_literal(record.parent_title)},
+            {record.page_start if record.page_start is not None else 'NULL'},
+            {record.page_end if record.page_end is not None else 'NULL'}
+        )
+        ON CONFLICT (edition_id, unit_key) DO UPDATE SET
+            parent_title = EXCLUDED.parent_title,
+            page_start = EXCLUDED.page_start,
+            page_end = EXCLUDED.page_end
+        RETURNING logical_unit_id
+    ),
+    upsert_chunk AS (
+        INSERT INTO knowledge.semantic_chunk (
+            chunk_id, logical_unit_id, edition_id, chunker_version,
+            chunk_index, content, token_count, content_hash,
+            section_id, subsection_id, parent_title, page_start, page_end,
+            is_active
+        )
+        SELECT
+            '{record.chunk_uuid}'::uuid,
+            logical_unit_id,
+            '{edition_id}'::uuid,
+            {_sql_literal(record.chunker_version)},
+            {record.chunk_index},
+            {_sql_literal(record.content)},
+            {record.token_count},
+            {_sql_literal(record.content_hash)},
+            {record.section_id if record.section_id is not None else 'NULL'},
+            {_sql_literal(record.subsection_id)},
+            {_sql_literal(record.parent_title)},
+            {record.page_start if record.page_start is not None else 'NULL'},
+            {record.page_end if record.page_end is not None else 'NULL'},
+            true
+        FROM upsert_unit
+        ON CONFLICT (logical_unit_id, chunk_index, chunker_version) DO UPDATE SET
+            content = EXCLUDED.content,
+            token_count = EXCLUDED.token_count,
+            content_hash = EXCLUDED.content_hash,
+            section_id = EXCLUDED.section_id,
+            subsection_id = EXCLUDED.subsection_id,
+            parent_title = EXCLUDED.parent_title,
+            page_start = EXCLUDED.page_start,
+            page_end = EXCLUDED.page_end,
+            is_active = true
+        RETURNING chunk_id
+    ),
+    upsert_provenance AS (
+        INSERT INTO knowledge.chunk_provenance (
+            chunk_id, checkpoint_db, processed_chunk_id, source_path
+        )
+        SELECT upsert_chunk.chunk_id, p.checkpoint_db, p.processed_chunk_id, p.source_path
+        FROM upsert_chunk
+        CROSS JOIN (VALUES {', '.join(provenance_values)}) AS p(
+            checkpoint_db, processed_chunk_id, source_path
+        )
+        ON CONFLICT (chunk_id, checkpoint_db, processed_chunk_id) DO UPDATE SET
+            source_path = EXCLUDED.source_path
+        RETURNING chunk_id
+    )
+    INSERT INTO knowledge.semantic_chunk_embedding (
+        chunk_id, model_id, edition_id, embedding, embedding_status, embedded_at
+    )
+    SELECT upsert_chunk.chunk_id, '{context.model_id}'::uuid, '{edition_id}'::uuid,
+           '{vector_literal}'::vector, 'indexed', now()
+    FROM upsert_chunk
+    ON CONFLICT (chunk_id, model_id) DO UPDATE SET
+        embedding = EXCLUDED.embedding,
+        embedding_status = 'indexed',
+        embedded_at = now();
+    """
+    _run_sql_via_cli(sql)

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/pipeline.py

@@ -0,0 +1,277 @@
+"""End-to-end semantic chunking and Supabase embedding upload pipeline."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import logging
+from pathlib import Path
+
+from .assemble import assemble_logical_units
+from .chunker import chunk_logical_unit, embedding_text
+from .config import (
+    BOOK_DB_FILES,
+    CHUNKER_VERSION,
+    DEFAULT_CHUNKER_CONFIG,
+    DEFAULT_ENV_PATH,
+    resolve_corpus_db_path,
+)
+from .embedding import GemmaEmbedder, get_embedder
+from .sqlite_store import (
+    count_semantic_chunks,
+    finish_chunking_run,
+    init_semantic_tables,
+    load_pending_chunks,
+    load_processed_chunks,
+    mark_embedding_status,
+    start_chunking_run,
+    upsert_semantic_chunk,
+)
+from .supabase_uploader import build_supabase_context, upload_semantic_chunk
+
+try:
+    from .pg_uploader import build_postgres_context, upload_semantic_chunk_pg
+except ImportError:
+    build_postgres_context = None
+    upload_semantic_chunk_pg = None
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+async def run_chunk_stage(
+    book_id: str,
+    embedder: GemmaEmbedder,
+    force: bool = False,
+) -> tuple[int, int]:
+    """Chunk one book from processed_chunks into local semantic_chunks."""
+    db_path = resolve_corpus_db_path(book_id)
+    if not db_path.exists():
+        raise FileNotFoundError(f"Checkpoint database not found: {db_path}")
+
+    await init_semantic_tables(db_path)
+    run_id = await start_chunking_run(db_path, book_id, CHUNKER_VERSION)
+    units_processed = 0
+    chunks_written = 0
+
+    try:
+        processed_rows = await load_processed_chunks(db_path)
+        logical_units = assemble_logical_units(processed_rows, book_id=book_id)
+
+        # print("check the logical units", len(logical_units))
+        for unit in logical_units:
+            records = chunk_logical_unit(
+                unit,
+                embedder=embedder,
+                config=DEFAULT_CHUNKER_CONFIG,
+                chunker_version=CHUNKER_VERSION,
+            )
+            units_processed += 1
+            for record in records:
+                if not force:
+                    from .sqlite_store import chunk_exists
+
+                    exists = await chunk_exists(
+                        db_path,
+                        record.logical_unit_id,
+                        record.chunk_index,
+                        record.chunker_version,
+                        record.book_id,
+                    )
+                    if exists:
+                        continue
+                await upsert_semantic_chunk(db_path, record)
+                chunks_written += 1
+        # print("check the units processed and chunks written", units_processed, chunks_written)
+        await finish_chunking_run(
+            db_path,
+            run_id,
+            units_processed=units_processed,
+            chunks_written=chunks_written,
+        )
+        table_count = await count_semantic_chunks(db_path, book_id, CHUNKER_VERSION)
+        if chunks_written > 0 and chunks_written != table_count:
+            logger.warning(
+                "Chunk count mismatch for %s: chunks_written=%s semantic_chunks=%s "
+                "(possible logical_unit_id collision or stale rows; re-run with --force "
+                "after clearing old rows for this chunker_version)",
+                book_id,
+                chunks_written,
+                table_count,
+            )
+        logger.info(
+            "Chunk stage complete for %s: units=%s chunks=%s table_rows=%s",
+            book_id,
+            units_processed,
+            chunks_written,
+            table_count,
+        )
+        return units_processed, chunks_written
+    except Exception as exc:
+        await finish_chunking_run(
+            db_path,
+            run_id,
+            units_processed=units_processed,
+            chunks_written=chunks_written,
+            status="failed",
+            error_message=str(exc),
+        )
+        raise
+
+
+async def run_upload_stage(
+    book_id: str,
+    embedder: GemmaEmbedder,
+    env_path: Path = DEFAULT_ENV_PATH,
+) -> int:
+    """Embed pending semantic chunks and upload them to Supabase."""
+    db_path = resolve_corpus_db_path(book_id)
+    pending = await load_pending_chunks(db_path, book_id, CHUNKER_VERSION)
+    if not pending:
+        logger.info("No pending chunks to upload for %s", book_id)
+        return 0
+
+    processed_rows = await load_processed_chunks(db_path)
+    source_paths = {row_id: source_path for row_id, source_path, _ in processed_rows}
+    checkpoint_db = BOOK_DB_FILES[book_id]
+
+    from .supabase_uploader import load_supabase_env
+
+    load_supabase_env(env_path)
+
+    rest_context = None
+    pg_context = None
+
+    if build_postgres_context is not None:
+        try:
+            pg_context = build_postgres_context(book_ids=[book_id])
+            logger.info("Using PostgreSQL upload mode: %s", pg_context.mode)
+        except Exception as exc:
+            logger.warning("PostgreSQL upload unavailable: %s", exc)
+            pg_context = None
+
+    if pg_context is None:
+        try:
+            rest_context = build_supabase_context(book_ids=[book_id])
+            logger.info("Using Supabase REST upload")
+        except Exception as exc:
+            raise RuntimeError(
+                "Upload failed. Link Supabase CLI (`supabase link`) for PostgreSQL upload, "
+                "or expose the `knowledge` schema and provide SUPABASE_SERVICE_ROLE_KEY."
+            ) from exc
+
+    uploaded = 0
+    for record in pending:
+        try:
+            vector = embedder.embed_document(
+                embedding_text(record),
+                title=record.parent_title,
+            )
+            if pg_context is not None:
+                upload_semantic_chunk_pg(
+                    pg_context,
+                    record,
+                    vector,
+                    checkpoint_db=checkpoint_db,
+                    source_paths=source_paths,
+                )
+            else:
+                upload_semantic_chunk(
+                    rest_context,
+                    record,
+                    vector,
+                    checkpoint_db=checkpoint_db,
+                    source_paths=source_paths,
+                )
+            await mark_embedding_status(db_path, record.chunk_uuid, "done")
+            uploaded += 1
+            if uploaded % 10 == 0:
+                logger.info("Uploaded %s/%s chunks for %s", uploaded, len(pending), book_id)
+        except Exception as exc:
+            logger.exception(
+                "Failed to upload chunk %s (%s): %s",
+                record.chunk_uuid,
+                record.logical_unit_id,
+                exc,
+            )
+            await mark_embedding_status(db_path, record.chunk_uuid, "failed")
+
+    logger.info("Upload stage complete for %s: uploaded=%s", book_id, uploaded)
+    return uploaded
+
+
+async def run_pipeline(
+    book_ids: list[str],
+    chunk: bool = True,
+    upload: bool = True,
+    force: bool = False,
+    env_path: Path = DEFAULT_ENV_PATH,
+) -> None:
+    """Run chunk and/or upload stages for the requested books."""
+    embedder = get_embedder()
+
+    for book_id in book_ids:
+        logger.info("=== Processing book: %s ===", book_id)
+        if chunk:
+            await run_chunk_stage(book_id, embedder, force=force)
+        if upload:
+            await run_upload_stage(book_id, embedder, env_path=env_path)
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse CLI arguments for the ingestion pipeline."""
+    parser = argparse.ArgumentParser(
+        description="Semantic chunking and Supabase embedding upload pipeline.",
+    )
+    parser.add_argument(
+        "--books",
+        nargs="+",
+        choices=sorted(BOOK_DB_FILES.keys()),
+        default=["mor"],
+        help="Books to process (default: mor).",
+    )
+    parser.add_argument(
+        "--chunk-only",
+        action="store_true",
+        help="Run Stage 3 chunking only (SQLite staging).",
+    )
+    parser.add_argument(
+        "--upload-only",
+        action="store_true",
+        help="Run Stage 4 embedding upload only.",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Re-chunk even when chunk slots already exist.",
+    )
+    parser.add_argument(
+        "--env-path",
+        type=Path,
+        default=DEFAULT_ENV_PATH,
+        help="Path to .env file containing Supabase credentials.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    """CLI entrypoint."""
+    args = parse_args()
+    chunk = not args.upload_only
+    upload = not args.chunk_only
+    asyncio.run(
+        run_pipeline(
+            book_ids=args.books,
+            chunk=chunk,
+            upload=upload,
+            force=args.force,
+            env_path=args.env_path,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/requirements.txt

@@ -0,0 +1,14 @@
+# Semantic chunking + Supabase embedding upload pipeline dependencies.
+# Install inside conda env `vkist_ultra` or an equivalent Python 3.11+ environment.
+
+aiosqlite>=0.20.0
+langchain-text-splitters>=0.3.0
+numpy>=1.26.0
+onnxruntime>=1.19.0
+optimum[onnxruntime]>=1.23.0
+python-dotenv>=1.0.0
+supabase>=2.10.0
+psycopg2-binary>=2.9.9
+pgvector>=0.3.0
+torch>=2.4.0
+transformers>=4.45.0

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/sqlite_store.py

@@ -0,0 +1,249 @@
+"""SQLite staging for semantic chunk checkpoint tables."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import aiosqlite
+
+from .chunker import SemanticChunkRecord
+
+logger = logging.getLogger(__name__)
+
+SEMANTIC_CHUNKS_DDL = """
+CREATE TABLE IF NOT EXISTS semantic_chunks (
+    id                      INTEGER PRIMARY KEY AUTOINCREMENT,
+    chunk_uuid              TEXT NOT NULL UNIQUE,
+    logical_unit_id         TEXT NOT NULL,
+    chunk_index             INTEGER NOT NULL,
+    content                 TEXT NOT NULL,
+    token_count             INTEGER NOT NULL,
+    book_id                 TEXT NOT NULL,
+    section_id              INTEGER,
+    subsection_id           TEXT,
+    parent_title            TEXT,
+    page_start              INTEGER,
+    page_end                INTEGER,
+    source_extraction_ids   TEXT NOT NULL,
+    chunker_version         TEXT NOT NULL,
+    embedding_status        TEXT NOT NULL DEFAULT 'pending',
+    content_hash            TEXT NOT NULL,
+    created_at              TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (logical_unit_id, chunk_index, chunker_version, book_id)
+);
+"""
+
+CHUNKING_RUNS_DDL = """
+CREATE TABLE IF NOT EXISTS chunking_runs (
+    id              INTEGER PRIMARY KEY AUTOINCREMENT,
+    book_id         TEXT NOT NULL,
+    chunker_version TEXT NOT NULL,
+    started_at      TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    finished_at     TIMESTAMP,
+    status          TEXT NOT NULL DEFAULT 'running',
+    units_processed INTEGER DEFAULT 0,
+    chunks_written  INTEGER DEFAULT 0,
+    error_message   TEXT
+);
+"""
+
+
+async def init_semantic_tables(db_path: Path) -> None:
+    """Ensure semantic chunk staging tables exist in the checkpoint database."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(SEMANTIC_CHUNKS_DDL)
+        await db.execute(CHUNKING_RUNS_DDL)
+        await db.commit()
+
+
+async def load_processed_chunks(db_path: Path) -> list[tuple[int, str, str]]:
+    """Load all processed_chunks rows from a checkpoint database."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            "SELECT id, source_path, markdown_content FROM processed_chunks ORDER BY id"
+        )
+        return await cursor.fetchall()
+
+
+async def chunk_exists(
+    db_path: Path,
+    logical_unit_id: str,
+    chunk_index: int,
+    chunker_version: str,
+    book_id: str,
+) -> bool:
+    """Return True when an identical chunk slot already exists."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT 1 FROM semantic_chunks
+            WHERE logical_unit_id = ? AND chunk_index = ?
+              AND chunker_version = ? AND book_id = ?
+            LIMIT 1
+            """,
+            (logical_unit_id, chunk_index, chunker_version, book_id),
+        )
+        row = await cursor.fetchone()
+        return row is not None
+
+
+async def upsert_semantic_chunk(db_path: Path, record: SemanticChunkRecord) -> None:
+    """Insert or replace one semantic chunk row."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            INSERT INTO semantic_chunks (
+                chunk_uuid, logical_unit_id, chunk_index, content, token_count,
+                book_id, section_id, subsection_id, parent_title, page_start,
+                page_end, source_extraction_ids, chunker_version, embedding_status,
+                content_hash
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?)
+            ON CONFLICT(logical_unit_id, chunk_index, chunker_version, book_id)
+            DO UPDATE SET
+                chunk_uuid = excluded.chunk_uuid,
+                content = excluded.content,
+                token_count = excluded.token_count,
+                section_id = excluded.section_id,
+                subsection_id = excluded.subsection_id,
+                parent_title = excluded.parent_title,
+                page_start = excluded.page_start,
+                page_end = excluded.page_end,
+                source_extraction_ids = excluded.source_extraction_ids,
+                content_hash = excluded.content_hash,
+                embedding_status = 'pending'
+            """,
+            (
+                record.chunk_uuid,
+                record.logical_unit_id,
+                record.chunk_index,
+                record.content,
+                record.token_count,
+                record.book_id,
+                record.section_id,
+                record.subsection_id,
+                record.parent_title,
+                record.page_start,
+                record.page_end,
+                json.dumps(record.source_extraction_ids),
+                record.chunker_version,
+                record.content_hash,
+            ),
+        )
+        await db.commit()
+
+
+async def load_pending_chunks(
+    db_path: Path,
+    book_id: str,
+    chunker_version: str,
+) -> list[SemanticChunkRecord]:
+    """Load semantic chunks that still need embedding upload."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT chunk_uuid, logical_unit_id, chunk_index, content, token_count,
+                   book_id, section_id, subsection_id, parent_title, page_start,
+                   page_end, source_extraction_ids, chunker_version, content_hash
+            FROM semantic_chunks
+            WHERE book_id = ? AND chunker_version = ?
+              AND embedding_status IN ('pending', 'failed')
+            ORDER BY logical_unit_id, chunk_index
+            """,
+            (book_id, chunker_version),
+        )
+        rows = await cursor.fetchall()
+
+    records: list[SemanticChunkRecord] = []
+    for row in rows:
+        records.append(
+            SemanticChunkRecord(
+                chunk_uuid=row[0],
+                logical_unit_id=row[1],
+                chunk_index=row[2],
+                content=row[3],
+                token_count=row[4],
+                book_id=row[5],
+                section_id=row[6],
+                subsection_id=row[7],
+                parent_title=row[8],
+                page_start=row[9],
+                page_end=row[10],
+                source_extraction_ids=json.loads(row[11]),
+                chunker_version=row[12],
+                content_hash=row[13],
+            )
+        )
+    return records
+
+
+async def mark_embedding_status(
+    db_path: Path,
+    chunk_uuid: str,
+    status: str,
+) -> None:
+    """Update embedding_status for one staged chunk."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            "UPDATE semantic_chunks SET embedding_status = ? WHERE chunk_uuid = ?",
+            (status, chunk_uuid),
+        )
+        await db.commit()
+
+
+async def start_chunking_run(db_path: Path, book_id: str, chunker_version: str) -> int:
+    """Create a chunking run audit row and return its id."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            INSERT INTO chunking_runs (book_id, chunker_version, status)
+            VALUES (?, ?, 'running')
+            """,
+            (book_id, chunker_version),
+        )
+        await db.commit()
+        return cursor.lastrowid
+
+
+async def count_semantic_chunks(
+    db_path: Path,
+    book_id: str,
+    chunker_version: str,
+) -> int:
+    """Return the number of staged semantic chunks for one book and chunker version."""
+    async with aiosqlite.connect(db_path) as db:
+        cursor = await db.execute(
+            """
+            SELECT COUNT(*) FROM semantic_chunks
+            WHERE book_id = ? AND chunker_version = ?
+            """,
+            (book_id, chunker_version),
+        )
+        row = await cursor.fetchone()
+        return int(row[0]) if row else 0
+
+
+async def finish_chunking_run(
+    db_path: Path,
+    run_id: int,
+    units_processed: int,
+    chunks_written: int,
+    status: str = "succeeded",
+    error_message: str | None = None,
+) -> None:
+    """Finalize a chunking run audit row."""
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            UPDATE chunking_runs
+            SET finished_at = CURRENT_TIMESTAMP,
+                units_processed = ?,
+                chunks_written = ?,
+                status = ?,
+                error_message = ?
+            WHERE id = ?
+            """,
+            (units_processed, chunks_written, status, error_message, run_id),
+        )
+        await db.commit()

workspace/sprint_1_2/CODEBASE/knowledge/implementation/ingestion/supabase_uploader.py

@@ -0,0 +1,306 @@
+"""Supabase upsert for semantic chunks and EmbeddingGemma vectors."""
+
+from __future__ import annotations
+
+import logging
+import os
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from dotenv import load_dotenv
+from supabase import Client, create_client
+
+from .chunker import SemanticChunkRecord
+from .config import (
+    CHUNKER_VERSION,
+    DEFAULT_ENV_PATH,
+    EDITION_LABEL,
+    EMBEDDING_DIMENSIONS,
+    EMBEDDING_MODEL_NAME,
+    EMBEDDING_MODEL_VERSION,
+)
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class SupabaseContext:
+    """Resolved Supabase identifiers required for ingestion upserts."""
+
+    client: Client
+    edition_ids: dict[str, str]
+    model_id: str
+
+
+def load_supabase_env(env_path: Path = DEFAULT_ENV_PATH) -> None:
+    """Load Supabase credentials from the project secrets file."""
+    if env_path.exists():
+        load_dotenv(env_path, override=False)
+
+
+def _resolve_supabase_key() -> str:
+    """Resolve the Supabase API key, preferring service role when available."""
+    service_role = os.getenv("SUPABASE_SERVICE_ROLE_KEY")
+    if service_role:
+        return service_role
+
+    publishable = os.getenv("SUPABASE_PUBLISHABLE_KEY")
+    if publishable:
+        logger.warning(
+            "Using SUPABASE_PUBLISHABLE_KEY for ingestion. "
+            "Writes require service_role or INSERT policies on knowledge.* tables."
+        )
+        return publishable
+
+    raise RuntimeError(
+        "Missing Supabase credentials. Set SUPABASE_URL and "
+        "SUPABASE_PUBLISHABLE_KEY (or SUPABASE_SERVICE_ROLE_KEY)."
+    )
+
+
+def create_supabase_client() -> Client:
+    """Create a Supabase client from environment variables."""
+    load_supabase_env()
+    url = os.getenv("SUPABASE_URL")
+    if not url:
+        raise RuntimeError("SUPABASE_URL is not set.")
+    key = _resolve_supabase_key()
+    return create_client(url, key)
+
+
+def _schema(client: Client):
+    return client.schema("knowledge")
+
+
+def _single_row(response: Any, label: str) -> dict[str, Any]:
+    data = response.data
+    if not data:
+        raise RuntimeError(f"No rows returned for {label}.")
+    return data[0]
+
+
+def _get_corpus_source_id(client: Client, book_id: str) -> str:
+    response = (
+        _schema(client)
+        .table("corpus_source")
+        .select("corpus_source_id")
+        .eq("book_id", book_id)
+        .limit(1)
+        .execute()
+    )
+    return _single_row(response, f"corpus_source[{book_id}]")["corpus_source_id"]
+
+
+def _get_embedding_model_id(client: Client) -> str:
+    response = (
+        _schema(client)
+        .table("embedding_model")
+        .select("model_id")
+        .eq("model_name", EMBEDDING_MODEL_NAME)
+        .eq("model_version", EMBEDDING_MODEL_VERSION)
+        .limit(1)
+        .execute()
+    )
+    return _single_row(response, "embedding_model")["model_id"]
+
+
+def ensure_active_edition(client: Client, book_id: str) -> str:
+    """Return the active edition_id for a book, creating one when missing."""
+    corpus_source_id = _get_corpus_source_id(client, book_id)
+    response = (
+        _schema(client)
+        .table("corpus_edition")
+        .select("edition_id")
+        .eq("corpus_source_id", corpus_source_id)
+        .eq("status", "active")
+        .limit(1)
+        .execute()
+    )
+    if response.data:
+        return response.data[0]["edition_id"]
+
+    insert_response = (
+        _schema(client)
+        .table("corpus_edition")
+        .insert(
+            {
+                "corpus_source_id": corpus_source_id,
+                "edition_label": EDITION_LABEL,
+                "status": "active",
+            }
+        )
+        .execute()
+    )
+    edition_id = _single_row(insert_response, f"corpus_edition[{book_id}]")["edition_id"]
+    logger.info("Created active corpus_edition for %s: %s", book_id, edition_id)
+    return edition_id
+
+
+def build_supabase_context(
+    client: Client | None = None,
+    book_ids: list[str] | None = None,
+) -> SupabaseContext:
+    """Resolve edition and model identifiers for a set of books."""
+    resolved_client = client or create_supabase_client()
+    target_books = book_ids or ["ado", "tny", "oho", "mor"]
+
+    try:
+        edition_ids = {
+            book_id: ensure_active_edition(resolved_client, book_id)
+            for book_id in target_books
+        }
+        model_id = _get_embedding_model_id(resolved_client)
+    except Exception as exc:
+        raise RuntimeError(
+            "Supabase REST upload failed. Ensure the hosted project exposes the "
+            "`knowledge` schema (Dashboard → API → Exposed schemas) and provide "
+            "SUPABASE_SERVICE_ROLE_KEY, or set SUPABASE_DB_URL for direct PostgreSQL upload."
+        ) from exc
+
+    return SupabaseContext(
+        client=resolved_client,
+        edition_ids=edition_ids,
+        model_id=model_id,
+    )
+
+
+def _upsert_logical_unit(
+    client: Client,
+    edition_id: str,
+    record: SemanticChunkRecord,
+) -> str:
+    payload = {
+        "edition_id": edition_id,
+        "unit_key": record.logical_unit_id,
+        "parent_title": record.parent_title,
+        "page_start": record.page_start,
+        "page_end": record.page_end,
+    }
+    response = (
+        _schema(client)
+        .table("logical_unit")
+        .upsert(payload, on_conflict="edition_id,unit_key")
+        .select("logical_unit_id")
+        .execute()
+    )
+    return _single_row(response, f"logical_unit[{record.logical_unit_id}]")["logical_unit_id"]
+
+
+def _upsert_semantic_chunk(
+    client: Client,
+    edition_id: str,
+    logical_unit_uuid: str,
+    record: SemanticChunkRecord,
+) -> str:
+    payload = {
+        "chunk_id": record.chunk_uuid,
+        "logical_unit_id": logical_unit_uuid,
+        "edition_id": edition_id,
+        "chunker_version": record.chunker_version,
+        "chunk_index": record.chunk_index,
+        "content": record.content,
+        "token_count": record.token_count,
+        "content_hash": record.content_hash,
+        "section_id": record.section_id,
+        "subsection_id": record.subsection_id,
+        "parent_title": record.parent_title,
+        "page_start": record.page_start,
+        "page_end": record.page_end,
+        "is_active": True,
+    }
+    response = (
+        _schema(client)
+        .table("semantic_chunk")
+        .upsert(payload, on_conflict="logical_unit_id,chunk_index,chunker_version")
+        .select("chunk_id")
+        .execute()
+    )
+    return _single_row(response, f"semantic_chunk[{record.chunk_uuid}]")["chunk_id"]
+
+
+def _upsert_chunk_provenance(
+    client: Client,
+    chunk_id: str,
+    checkpoint_db: str,
+    source_extraction_ids: list[int],
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    rows = []
+    for extraction_id in source_extraction_ids:
+        rows.append(
+            {
+                "chunk_id": chunk_id,
+                "checkpoint_db": checkpoint_db,
+                "processed_chunk_id": extraction_id,
+                "source_path": (source_paths or {}).get(
+                    extraction_id,
+                    f"processed_chunks:{extraction_id}",
+                ),
+            }
+        )
+    _schema(client).table("chunk_provenance").upsert(
+        rows,
+        on_conflict="chunk_id,checkpoint_db,processed_chunk_id",
+    ).execute()
+
+
+def _format_vector(vector: np.ndarray) -> list[float]:
+    if vector.shape[0] != EMBEDDING_DIMENSIONS:
+        raise ValueError(
+            f"Expected {EMBEDDING_DIMENSIONS}-d embedding, got {vector.shape[0]}"
+        )
+    return [float(value) for value in vector.tolist()]
+
+
+def _upsert_embedding(
+    client: Client,
+    chunk_id: str,
+    edition_id: str,
+    model_id: str,
+    vector: np.ndarray,
+) -> None:
+    payload = {
+        "chunk_id": chunk_id,
+        "model_id": model_id,
+        "edition_id": edition_id,
+        "embedding": _format_vector(vector),
+        "embedding_status": "indexed",
+    }
+    _schema(client).table("semantic_chunk_embedding").upsert(
+        payload,
+        on_conflict="chunk_id,model_id",
+    ).execute()
+
+
+def upload_semantic_chunk(
+    context: SupabaseContext,
+    record: SemanticChunkRecord,
+    vector: np.ndarray,
+    checkpoint_db: str,
+    source_paths: dict[int, str] | None = None,
+) -> None:
+    """Upsert one chunk and its embedding vector to Supabase."""
+    edition_id = context.edition_ids[record.book_id]
+    logical_unit_uuid = _upsert_logical_unit(context.client, edition_id, record)
+    chunk_id = _upsert_semantic_chunk(
+        context.client,
+        edition_id,
+        logical_unit_uuid,
+        record,
+    )
+    _upsert_chunk_provenance(
+        context.client,
+        chunk_id,
+        checkpoint_db,
+        record.source_extraction_ids,
+        source_paths=source_paths,
+    )
+    _upsert_embedding(
+        context.client,
+        chunk_id,
+        edition_id,
+        context.model_id,
+        vector,
+    )

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/.gitignore

@@ -0,0 +1,8 @@
+# Supabase
+.branches
+.temp
+
+# dotenvx
+.env.keys
+.env.local
+.env.*.local

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/README.md

@@ -0,0 +1,108 @@
+# Knowledge Supabase — Semantic Vector DB
+
+PostgreSQL + pgvector schema for clinical textbook RAG, derived from [`spec/pg_semantic_vector_db/er_diagram.md`](../spec/pg_semantic_vector_db/er_diagram.md).
+
+## Layout
+
+```
+knowledge/supabase/
+├── config.toml
+├── README.md
+└── migrations/
+    ├── 20260705000000_semantic_vector_db_schema.sql   # tables + HNSW index
+    ├── 20260705000001_semantic_vector_db_seed.sql     # corpus_source, chunker, model
+    ├── 20260705000002_semantic_vector_db_rls_and_rpc.sql  # RLS + match_semantic_chunks()
+    ├── 20260705000003_expose_knowledge_api_schema.sql # PostgREST grants
+    ├── 20260705000004_chunker_profile_v2.sql          # header+semantic_v2 profile
+    ├── 20260705000005_corpus_bibliography.sql         # bibliography columns + contributors
+    └── 20260705000006_corpus_bibliography_seed.sql    # seed from corpus/source_metadata.md
+```
+
+All application tables live in the **`knowledge`** schema (not `public`).
+
+## Apply to hosted Supabase
+
+### Option A — Supabase CLI (recommended)
+
+```bash
+cd workspace/sprint_1_2/CODEBASE/knowledge/supabase
+
+# Link your project (once)
+supabase link --project-ref <your-project-ref>
+
+# Push migrations
+supabase db push
+```
+
+### Option B — SQL Editor
+
+Run each migration file in order in the Supabase Dashboard → SQL Editor.
+
+## Local development
+
+```bash
+cd workspace/sprint_1_2/CODEBASE/knowledge/supabase
+supabase start
+supabase db reset   # applies all migrations + seed
+```
+
+Local Postgres: `postgresql://postgres:postgres@localhost:54322/postgres`
+
+## Environment variables
+
+Add to your backend `.env`:
+
+```bash
+SUPABASE_URL=https://<project-ref>.supabase.co
+SUPABASE_SERVICE_ROLE_KEY=<service-role-key>   # ingestion + RAG backend only
+# Optional: for authenticated read policies
+SUPABASE_ANON_KEY=<anon-key>
+```
+
+**Never expose `SUPABASE_SERVICE_ROLE_KEY` to the frontend.**
+
+## Runtime RAG query
+
+After Stage 4 embed upsert, call the RPC from the backend:
+
+```sql
+SELECT * FROM knowledge.match_semantic_chunks(
+  query_embedding := $1::extensions.vector(768),
+  match_count     := 5,
+  filter_book_ids := ARRAY['mor', 'oho']
+);
+```
+
+Active-edition filter is enforced inside the function:
+
+- `corpus_edition.status = 'active'`
+- `semantic_chunk.is_active = true`
+- `semantic_chunk_embedding.embedding_status = 'indexed'`
+
+## Ingestion upsert flow (Stage 4)
+
+1. Ensure `corpus_edition` exists for the book (status `active`).
+2. Upsert `logical_unit` rows per assembled unit.
+3. Upsert `semantic_chunk` rows (idempotent on `logical_unit_id + chunk_index + chunker_version`).
+4. Insert `chunk_provenance` rows from SQLite `source_extraction_ids`.
+5. Upsert `semantic_chunk_embedding` with `embedding_status = 'indexed'` and the 768-d vector.
+
+## Verify after migration
+
+```sql
+SELECT book_id, full_title, subject_area FROM knowledge.corpus_source;
+SELECT book_id, edition_label, publisher_name, published_year
+FROM knowledge.v_corpus_citation;
+SELECT cs.book_id, cc.role, cc.full_name
+FROM knowledge.corpus_contributor cc
+JOIN knowledge.corpus_source cs ON cs.corpus_source_id = cc.corpus_source_id
+ORDER BY cs.book_id, cc.sort_order;
+SELECT chunker_version FROM knowledge.chunker_profile;
+SELECT model_name, dimensions FROM knowledge.embedding_model WHERE is_active;
+```
+
+## References
+
+- [`spec/pg_semantic_vector_db/er_diagram.md`](../spec/pg_semantic_vector_db/er_diagram.md)
+- [`spec/pg_semantic_vector_db/supabase_schema.md`](../spec/pg_semantic_vector_db/supabase_schema.md)
+- [`spec/ingestion/schema.md`](../spec/ingestion/schema.md) — SQLite staging → PG mapping

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/config.toml

@@ -0,0 +1,414 @@
+# For detailed configuration reference documentation, visit:
+# https://supabase.com/docs/guides/local-development/cli/config
+# A string used to distinguish different Supabase projects on the same host. Defaults to the
+# working directory name when running `supabase init`.
+project_id = "supabase"
+
+[api]
+enabled = true
+# Port to use for the API URL.
+port = 54321
+# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API
+# endpoints. `public` and `graphql_public` schemas are included by default.
+schemas = ["public", "graphql_public", "knowledge"]
+# Extra schemas to add to the search_path of every request.
+extra_search_path = ["public", "extensions"]
+# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size
+# for accidental or malicious requests.
+max_rows = 1000
+# Controls whether new tables, views, sequences and functions created in the `public` schema by
+# `postgres` are reachable through the Data API roles (`anon`, `authenticated`, `service_role`)
+# without explicit GRANTs. When unset, new entities are NOT auto-exposed, matching the new cloud
+# default. Set to `true` to keep the legacy behaviour of auto-exposing new entities; this is
+# deprecated and the field is removed on 2026-10-30 once the always-revoked behaviour is permanent.
+# auto_expose_new_tables = true
+
+[api.tls]
+# Enable HTTPS endpoints locally using a self-signed certificate.
+enabled = false
+# Paths to self-signed certificate pair.
+# cert_path = "../certs/my-cert.pem"
+# key_path = "../certs/my-key.pem"
+
+[db]
+# Port to use for the local database URL.
+port = 54322
+# Port used by db diff command to initialize the shadow database.
+shadow_port = 54320
+# Maximum amount of time to wait for health check when starting the local database.
+health_timeout = "2m"
+# The database major version to use. This has to be the same as your remote database's. Run `SHOW
+# server_version;` on the remote database to check.
+major_version = 17
+
+[db.pooler]
+enabled = false
+# Port to use for the local connection pooler.
+port = 54329
+# Specifies when a server connection can be reused by other clients.
+# Configure one of the supported pooler modes: `transaction`, `session`.
+pool_mode = "transaction"
+# How many server connections to allow per user/database pair.
+default_pool_size = 20
+# Maximum number of client connections allowed.
+max_client_conn = 100
+
+# [db.vault]
+# secret_key = "env(SECRET_VALUE)"
+
+[db.migrations]
+# If disabled, migrations will be skipped during a db push or reset.
+enabled = true
+# Specifies an ordered list of schema files that describe your database.
+# Supports glob patterns relative to supabase directory: "./schemas/*.sql"
+schema_paths = []
+
+[db.seed]
+# If enabled, seeds the database after migrations during a db reset.
+enabled = true
+# Specifies an ordered list of seed files to load during db reset.
+# Supports glob patterns relative to supabase directory: "./seeds/*.sql"
+sql_paths = ["./seed.sql"]
+
+[db.network_restrictions]
+# Enable management of network restrictions.
+enabled = false
+# List of IPv4 CIDR blocks allowed to connect to the database.
+# Defaults to allow all IPv4 connections. Set empty array to block all IPs.
+allowed_cidrs = ["0.0.0.0/0"]
+# List of IPv6 CIDR blocks allowed to connect to the database.
+# Defaults to allow all IPv6 connections. Set empty array to block all IPs.
+allowed_cidrs_v6 = ["::/0"]
+
+# Uncomment to reject non-secure connections to the database.
+# [db.ssl_enforcement]
+# enabled = true
+
+[realtime]
+enabled = true
+# Bind realtime via either IPv4 or IPv6. (default: IPv4)
+# ip_version = "IPv6"
+# The maximum length in bytes of HTTP request headers. (default: 4096)
+# max_header_length = 4096
+
+[studio]
+enabled = true
+# Port to use for Supabase Studio.
+port = 54323
+# External URL of the API server that frontend connects to.
+api_url = "http://127.0.0.1"
+# OpenAI API Key to use for Supabase AI in the Supabase Studio.
+openai_api_key = "env(OPENAI_API_KEY)"
+
+# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they
+# are monitored, and you can view the emails that would have been sent from the web interface.
+[local_smtp]
+enabled = true
+# Port to use for the email testing server web interface.
+port = 54324
+# Uncomment to expose additional ports for testing user applications that send emails.
+# smtp_port = 54325
+# pop3_port = 54326
+# admin_email = "admin@email.com"
+# sender_name = "Admin"
+
+[storage]
+enabled = true
+# The maximum file size allowed (e.g. "5MB", "500KB").
+file_size_limit = "50MiB"
+
+# Uncomment to configure local storage buckets
+# [storage.buckets.images]
+# public = false
+# file_size_limit = "50MiB"
+# allowed_mime_types = ["image/png", "image/jpeg"]
+# objects_path = "./images"
+
+# Allow connections via S3 compatible clients
+[storage.s3_protocol]
+enabled = true
+
+# Image transformation API is available to Supabase Pro plan.
+# [storage.image_transformation]
+# enabled = true
+
+# Store analytical data in S3 for running ETL jobs over Iceberg Catalog
+# This feature is only available on the hosted platform.
+[storage.analytics]
+enabled = false
+max_namespaces = 5
+max_tables = 10
+max_catalogs = 2
+
+# Analytics Buckets is available to Supabase Pro plan.
+# [storage.analytics.buckets.my-warehouse]
+
+# Store vector embeddings in S3 for large and durable datasets
+[storage.vector]
+enabled = true
+max_buckets = 10
+max_indexes = 5
+
+# Vector Buckets is available to Supabase Pro plan.
+# [storage.vector.buckets.documents-openai]
+
+[auth]
+enabled = true
+# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used
+# in emails.
+site_url = "http://127.0.0.1:3000"
+# The public URL that Auth serves on. Defaults to the API external URL with `/auth/v1` appended.
+# external_url = ""
+# A list of *exact* URLs that auth providers are permitted to redirect to post authentication.
+additional_redirect_urls = ["https://127.0.0.1:3000"]
+# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week).
+jwt_expiry = 3600
+# JWT issuer URL. If not set, defaults to auth.external_url.
+# jwt_issuer = ""
+# Path to JWT signing key. DO NOT commit your signing keys file to git.
+# signing_keys_path = "./signing_keys.json"
+# If disabled, the refresh token will never expire.
+enable_refresh_token_rotation = true
+# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds.
+# Requires enable_refresh_token_rotation = true.
+refresh_token_reuse_interval = 10
+# Allow/disallow new user signups to your project.
+enable_signup = true
+# Allow/disallow anonymous sign-ins to your project.
+enable_anonymous_sign_ins = false
+# Allow/disallow testing manual linking of accounts
+enable_manual_linking = false
+# Passwords shorter than this value will be rejected as weak. Minimum 6, recommended 8 or more.
+minimum_password_length = 6
+# Passwords that do not meet the following requirements will be rejected as weak. Supported values
+# are: `letters_digits`, `lower_upper_letters_digits`, `lower_upper_letters_digits_symbols`
+password_requirements = ""
+
+# Configure passkey sign-ins.
+# [auth.passkey]
+# enabled = false
+
+# Configure WebAuthn relying party settings (required when passkey is enabled).
+# [auth.webauthn]
+# rp_display_name = "Supabase"
+# rp_id = "localhost"
+# rp_origins = ["http://127.0.0.1:3000"]
+
+[auth.rate_limit]
+# Number of emails that can be sent per hour. Requires auth.email.smtp to be enabled.
+email_sent = 2
+# Number of SMS messages that can be sent per hour. Requires auth.sms to be enabled.
+sms_sent = 30
+# Number of anonymous sign-ins that can be made per hour per IP address. Requires enable_anonymous_sign_ins = true.
+anonymous_users = 30
+# Number of sessions that can be refreshed in a 5 minute interval per IP address.
+token_refresh = 150
+# Number of sign up and sign-in requests that can be made in a 5 minute interval per IP address (excludes anonymous users).
+sign_in_sign_ups = 30
+# Number of OTP / Magic link verifications that can be made in a 5 minute interval per IP address.
+token_verifications = 30
+# Number of Web3 logins that can be made in a 5 minute interval per IP address.
+web3 = 30
+
+# Configure one of the supported captcha providers: `hcaptcha`, `turnstile`.
+# [auth.captcha]
+# enabled = true
+# provider = "hcaptcha"
+# secret = ""
+
+[auth.email]
+# Allow/disallow new user signups via email to your project.
+enable_signup = true
+# If enabled, a user will be required to confirm any email change on both the old, and new email
+# addresses. If disabled, only the new email is required to confirm.
+double_confirm_changes = true
+# If enabled, users need to confirm their email address before signing in.
+enable_confirmations = false
+# If enabled, users will need to reauthenticate or have logged in recently to change their password.
+secure_password_change = false
+# Controls the minimum amount of time that must pass before sending another signup confirmation or password reset email.
+max_frequency = "1s"
+# Number of characters used in the email OTP.
+otp_length = 6
+# Number of seconds before the email OTP expires (defaults to 1 hour).
+otp_expiry = 3600
+
+# Use a production-ready SMTP server
+# [auth.email.smtp]
+# enabled = true
+# host = "smtp.sendgrid.net"
+# port = 587
+# user = "apikey"
+# pass = "env(SENDGRID_API_KEY)"
+# admin_email = "admin@email.com"
+# sender_name = "Admin"
+
+# Uncomment to customize email template
+# [auth.email.template.invite]
+# subject = "You have been invited"
+# content_path = "./supabase/templates/invite.html"
+
+# Uncomment to customize notification email template
+# [auth.email.notification.password_changed]
+# enabled = true
+# subject = "Your password has been changed"
+# content_path = "./templates/password_changed_notification.html"
+
+[auth.sms]
+# Allow/disallow new user signups via SMS to your project.
+enable_signup = false
+# If enabled, users need to confirm their phone number before signing in.
+enable_confirmations = false
+# Template for sending OTP to users
+template = "Your code is {{ `{{ .Code }}` }}"
+# Controls the minimum amount of time that must pass before sending another sms otp.
+max_frequency = "5s"
+
+# Use pre-defined map of phone number to OTP for testing.
+# [auth.sms.test_otp]
+# 4152127777 = "123456"
+
+# Configure logged in session timeouts.
+# [auth.sessions]
+# Force log out after the specified duration.
+# timebox = "24h"
+# Force log out if the user has been inactive longer than the specified duration.
+# inactivity_timeout = "8h"
+
+# This hook runs before a new user is created and allows developers to reject the request based on the incoming user object.
+# [auth.hook.before_user_created]
+# enabled = true
+# uri = "pg-functions://postgres/auth/before-user-created-hook"
+
+# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used.
+# [auth.hook.custom_access_token]
+# enabled = true
+# uri = "pg-functions://<database>/<schema>/<hook_name>"
+
+# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`.
+[auth.sms.twilio]
+enabled = false
+account_sid = ""
+message_service_sid = ""
+# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead:
+auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)"
+
+# Multi-factor-authentication is available to Supabase Pro plan.
+[auth.mfa]
+# Control how many MFA factors can be enrolled at once per user.
+max_enrolled_factors = 10
+
+# Control MFA via App Authenticator (TOTP)
+[auth.mfa.totp]
+enroll_enabled = false
+verify_enabled = false
+
+# Configure MFA via Phone Messaging
+[auth.mfa.phone]
+enroll_enabled = false
+verify_enabled = false
+otp_length = 6
+template = "Your code is {{ `{{ .Code }}` }}"
+max_frequency = "5s"
+
+# Configure MFA via WebAuthn
+# [auth.mfa.web_authn]
+# enroll_enabled = true
+# verify_enabled = true
+
+# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`,
+# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`,
+# `twitter`, `x`, `slack`, `spotify`, `workos`, `zoom`.
+[auth.external.apple]
+enabled = false
+client_id = ""
+# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead:
+secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)"
+# Overrides the default auth callback URL derived from auth.external_url.
+redirect_uri = ""
+# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure,
+# or any other third-party OIDC providers.
+url = ""
+# If enabled, the nonce check will be skipped. Required for local sign in with Google auth.
+skip_nonce_check = false
+# If enabled, it will allow the user to successfully authenticate when the provider does not return an email address.
+email_optional = false
+
+# Allow Solana wallet holders to sign in to your project via the Sign in with Solana (SIWS, EIP-4361) standard.
+# You can configure "web3" rate limit in the [auth.rate_limit] section and set up [auth.captcha] if self-hosting.
+[auth.web3.solana]
+enabled = false
+
+# Use Firebase Auth as a third-party provider alongside Supabase Auth.
+[auth.third_party.firebase]
+enabled = false
+# project_id = "my-firebase-project"
+
+# Use Auth0 as a third-party provider alongside Supabase Auth.
+[auth.third_party.auth0]
+enabled = false
+# tenant = "my-auth0-tenant"
+# tenant_region = "us"
+
+# Use AWS Cognito (Amplify) as a third-party provider alongside Supabase Auth.
+[auth.third_party.aws_cognito]
+enabled = false
+# user_pool_id = "my-user-pool-id"
+# user_pool_region = "us-east-1"
+
+# Use Clerk as a third-party provider alongside Supabase Auth.
+[auth.third_party.clerk]
+enabled = false
+# Obtain from https://clerk.com/setup/supabase
+# domain = "example.clerk.accounts.dev"
+
+# OAuth server configuration
+[auth.oauth_server]
+# Enable OAuth server functionality
+enabled = false
+# Path for OAuth consent flow UI
+authorization_url_path = "/oauth/consent"
+# Allow dynamic client registration
+allow_dynamic_registration = false
+
+[edge_runtime]
+enabled = true
+# Supported request policies: `oneshot`, `per_worker`.
+# `per_worker` (default) — enables hot reload during local development.
+# `oneshot` — fallback mode if hot reload causes issues (e.g. in large repos or with symlinks).
+policy = "per_worker"
+# Port to attach the Chrome inspector for debugging edge functions.
+inspector_port = 8083
+# The Deno major version to use.
+deno_version = 2
+
+# [edge_runtime.secrets]
+# secret_key = "env(SECRET_VALUE)"
+
+[analytics]
+enabled = true
+port = 54327
+# Configure one of the supported backends: `postgres`, `bigquery`.
+backend = "postgres"
+
+# Experimental features may be deprecated any time
+[experimental]
+# Configures Postgres storage engine to use OrioleDB (S3)
+orioledb_version = ""
+# Configures S3 bucket URL, eg. <bucket_name>.s3-<region>.amazonaws.com
+s3_host = "env(S3_HOST)"
+# Configures S3 bucket region, eg. us-east-1
+s3_region = "env(S3_REGION)"
+# Configures AWS_ACCESS_KEY_ID for S3 bucket
+s3_access_key = "env(S3_ACCESS_KEY)"
+# Configures AWS_SECRET_ACCESS_KEY for S3 bucket
+s3_secret_key = "env(S3_SECRET_KEY)"
+
+# pg-delta is the schema diff engine for db diff / db pull / db remote commit.
+# Set enabled = false to fall back to the legacy migra engine.
+[experimental.pgdelta]
+enabled = true
+# Directory under `supabase/` where declarative files are written.
+# declarative_schema_path = "./database"
+# JSON string passed through to pg-delta SQL formatting.
+# format_options = "{\"keywordCase\":\"upper\",\"indent\":2,\"maxWidth\":80,\"commaStyle\":\"trailing\"}"

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000000_semantic_vector_db_schema.sql

@@ -0,0 +1,300 @@
+-- pg-Semantic-Vector-DB schema for Supabase
+-- Source: knowledge/spec/pg_semantic_vector_db/er_diagram.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Extensions & schema
+-- ---------------------------------------------------------------------------
+CREATE EXTENSION IF NOT EXISTS vector WITH SCHEMA public;
+CREATE SCHEMA IF NOT EXISTS knowledge;
+
+COMMENT ON SCHEMA knowledge IS
+  'Clinical textbook semantic chunks and EmbeddingGemma vectors for RAG retrieval.';
+
+-- ---------------------------------------------------------------------------
+-- corpus_source
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.corpus_source (
+    corpus_source_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    book_id          text NOT NULL,
+    display_name     text NOT NULL,
+    constraint_tier  text NOT NULL,
+    locale           text NOT NULL DEFAULT 'vi-VN',
+    created_at       timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT corpus_source_book_id_key UNIQUE (book_id),
+    CONSTRAINT corpus_source_constraint_tier_check
+        CHECK (constraint_tier IN ('loose', 'section', 'subsection'))
+);
+
+COMMENT ON TABLE knowledge.corpus_source IS
+  'Registry of ingestible textbook corpora (ADO, TNY, OHO, MOR).';
+
+-- ---------------------------------------------------------------------------
+-- corpus_edition
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.corpus_edition (
+    edition_id         uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    corpus_source_id   uuid NOT NULL REFERENCES knowledge.corpus_source (corpus_source_id)
+        ON DELETE RESTRICT,
+    edition_label      text NOT NULL,
+    source_pdf_sha256  text,
+    status             text NOT NULL DEFAULT 'draft',
+    effective_from     date,
+    effective_to       date,
+    created_at         timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT corpus_edition_status_check
+        CHECK (status IN ('draft', 'active', 'superseded', 'archived'))
+);
+
+CREATE UNIQUE INDEX uq_corpus_edition_one_active
+    ON knowledge.corpus_edition (corpus_source_id)
+    WHERE status = 'active';
+
+CREATE INDEX idx_corpus_edition_source_status
+    ON knowledge.corpus_edition (corpus_source_id, status);
+
+COMMENT ON TABLE knowledge.corpus_edition IS
+  'Versioned PDF snapshot; re-chunking and re-embedding are scoped to an edition.';
+
+-- ---------------------------------------------------------------------------
+-- chunker_profile
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.chunker_profile (
+    chunker_version text PRIMARY KEY,
+    description     text,
+    max_tokens      integer NOT NULL DEFAULT 800,
+    min_tokens      integer NOT NULL DEFAULT 120,
+    overlap_tokens  integer NOT NULL DEFAULT 64,
+    params_json     jsonb NOT NULL DEFAULT '{}'::jsonb,
+    created_at      timestamptz NOT NULL DEFAULT now()
+);
+
+COMMENT ON TABLE knowledge.chunker_profile IS
+  'Version registry for header + semantic chunking algorithm parameters.';
+
+-- ---------------------------------------------------------------------------
+-- embedding_model
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.embedding_model (
+    model_id       uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    model_name     text NOT NULL,
+    model_version  text NOT NULL,
+    dimensions     integer NOT NULL,
+    purpose        text NOT NULL DEFAULT 'rag',
+    is_active      boolean NOT NULL DEFAULT true,
+    registered_at  timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT embedding_model_purpose_check
+        CHECK (purpose IN ('rag')),
+    CONSTRAINT embedding_model_dimensions_check
+        CHECK (dimensions > 0),
+    CONSTRAINT embedding_model_name_version_key
+        UNIQUE (model_name, model_version)
+);
+
+COMMENT ON TABLE knowledge.embedding_model IS
+  'Registry of embedding models allowed to populate the RAG vector index.';
+
+-- ---------------------------------------------------------------------------
+-- structure_node (optional TOC tree)
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.structure_node (
+    structure_node_id        uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id               uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    parent_structure_node_id uuid REFERENCES knowledge.structure_node (structure_node_id)
+        ON DELETE CASCADE,
+    node_type                text NOT NULL,
+    node_key                 text NOT NULL,
+    title                    text,
+    page_start               integer,
+    page_end                 integer,
+    sort_order               integer NOT NULL DEFAULT 0,
+
+    CONSTRAINT structure_node_node_type_check
+        CHECK (node_type IN ('part', 'section', 'subsection')),
+    CONSTRAINT structure_node_edition_node_key_key
+        UNIQUE (edition_id, node_key)
+);
+
+CREATE INDEX idx_structure_node_edition_parent
+    ON knowledge.structure_node (edition_id, parent_structure_node_id);
+
+COMMENT ON TABLE knowledge.structure_node IS
+  'Optional document hierarchy (TOC); strongly populated for OHO/MOR.';
+
+-- ---------------------------------------------------------------------------
+-- logical_unit
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.logical_unit (
+    logical_unit_id   uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    structure_node_id uuid REFERENCES knowledge.structure_node (structure_node_id)
+        ON DELETE SET NULL,
+    unit_key          text NOT NULL,
+    parent_title      text,
+    page_start        integer,
+    page_end          integer,
+    assembled_at      timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT logical_unit_edition_unit_key_key
+        UNIQUE (edition_id, unit_key)
+);
+
+CREATE INDEX idx_logical_unit_edition
+    ON knowledge.logical_unit (edition_id);
+
+COMMENT ON TABLE knowledge.logical_unit IS
+  'Assembly boundary before chunking (sec_N, sub_X.Y, etc.).';
+
+-- ---------------------------------------------------------------------------
+-- semantic_chunk
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.semantic_chunk (
+    chunk_id         uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    logical_unit_id  uuid NOT NULL REFERENCES knowledge.logical_unit (logical_unit_id)
+        ON DELETE CASCADE,
+    edition_id       uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    chunker_version  text NOT NULL REFERENCES knowledge.chunker_profile (chunker_version)
+        ON DELETE RESTRICT,
+    chunk_index      integer NOT NULL,
+    content          text NOT NULL,
+    token_count      integer NOT NULL,
+    content_hash     text NOT NULL,
+    section_id       integer,
+    subsection_id    text,
+    parent_title     text,
+    page_start       integer,
+    page_end         integer,
+    is_active        boolean NOT NULL DEFAULT true,
+    created_at       timestamptz NOT NULL DEFAULT now(),
+
+    CONSTRAINT semantic_chunk_logical_unit_chunk_index_version_key
+        UNIQUE (logical_unit_id, chunk_index, chunker_version),
+    CONSTRAINT semantic_chunk_token_count_check
+        CHECK (token_count >= 0)
+);
+
+CREATE INDEX idx_semantic_chunk_edition_active
+    ON knowledge.semantic_chunk (edition_id, is_active);
+
+CREATE INDEX idx_semantic_chunk_chunker_version
+    ON knowledge.semantic_chunk (chunker_version);
+
+COMMENT ON TABLE knowledge.semantic_chunk IS
+  'Retrieval-ready text unit with citation metadata; chunk_id is the API citation key.';
+
+-- ---------------------------------------------------------------------------
+-- chunk_provenance
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.chunk_provenance (
+    provenance_id       uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id            uuid NOT NULL REFERENCES knowledge.semantic_chunk (chunk_id)
+        ON DELETE CASCADE,
+    checkpoint_db       text NOT NULL,
+    processed_chunk_id  bigint NOT NULL,
+    source_path         text NOT NULL,
+
+    CONSTRAINT chunk_provenance_chunk_checkpoint_source_key
+        UNIQUE (chunk_id, checkpoint_db, processed_chunk_id)
+);
+
+CREATE INDEX idx_chunk_provenance_checkpoint
+    ON knowledge.chunk_provenance (checkpoint_db, processed_chunk_id);
+
+COMMENT ON TABLE knowledge.chunk_provenance IS
+  'Traceability from semantic chunks back to SQLite processed_chunks rows.';
+
+-- ---------------------------------------------------------------------------
+-- semantic_chunk_embedding
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.semantic_chunk_embedding (
+    embedding_id      uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    chunk_id          uuid NOT NULL REFERENCES knowledge.semantic_chunk (chunk_id)
+        ON DELETE CASCADE,
+    model_id          uuid NOT NULL REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE RESTRICT,
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    embedding         public.vector(768),
+    embedding_status  text NOT NULL DEFAULT 'pending',
+    embedded_at       timestamptz,
+
+    CONSTRAINT semantic_chunk_embedding_chunk_model_key
+        UNIQUE (chunk_id, model_id),
+    CONSTRAINT semantic_chunk_embedding_status_check
+        CHECK (embedding_status IN ('pending', 'indexed', 'failed'))
+);
+
+CREATE INDEX idx_semantic_chunk_embedding_edition_status
+    ON knowledge.semantic_chunk_embedding (edition_id, embedding_status);
+
+-- HNSW index for cosine similarity search (PoC scale ~15K vectors)
+CREATE INDEX idx_semantic_chunk_embedding_hnsw
+    ON knowledge.semantic_chunk_embedding
+    USING hnsw (embedding public.vector_cosine_ops)
+    WITH (m = 16, ef_construction = 64);
+
+COMMENT ON TABLE knowledge.semantic_chunk_embedding IS
+  'EmbeddingGemma vectors; separated from semantic_chunk for re-embed without duplicating text.';
+
+-- ---------------------------------------------------------------------------
+-- ingestion_run
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.ingestion_run (
+    run_id              uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id          uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    chunker_version     text NOT NULL REFERENCES knowledge.chunker_profile (chunker_version)
+        ON DELETE RESTRICT,
+    model_id            uuid REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE SET NULL,
+    stage               text NOT NULL,
+    status              text NOT NULL DEFAULT 'running',
+    units_processed     integer NOT NULL DEFAULT 0,
+    chunks_written      integer NOT NULL DEFAULT 0,
+    embeddings_written  integer NOT NULL DEFAULT 0,
+    error_message       text,
+    started_at          timestamptz NOT NULL DEFAULT now(),
+    finished_at         timestamptz,
+
+    CONSTRAINT ingestion_run_stage_check
+        CHECK (stage IN ('chunk', 'embed', 'full')),
+    CONSTRAINT ingestion_run_status_check
+        CHECK (status IN ('running', 'succeeded', 'failed'))
+);
+
+CREATE INDEX idx_ingestion_run_edition_started
+    ON knowledge.ingestion_run (edition_id, started_at DESC);
+
+COMMENT ON TABLE knowledge.ingestion_run IS
+  'Audit log for chunk and embed pipeline executions.';
+
+-- ---------------------------------------------------------------------------
+-- vector_index_snapshot
+-- ---------------------------------------------------------------------------
+CREATE TABLE knowledge.vector_index_snapshot (
+    index_snapshot_id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    edition_id        uuid NOT NULL REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    model_id          uuid NOT NULL REFERENCES knowledge.embedding_model (model_id)
+        ON DELETE RESTRICT,
+    index_name        text NOT NULL,
+    hnsw_params       jsonb NOT NULL DEFAULT '{}'::jsonb,
+    is_active         boolean NOT NULL DEFAULT false,
+    built_at          timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE UNIQUE INDEX uq_vector_index_snapshot_one_active
+    ON knowledge.vector_index_snapshot (edition_id, model_id)
+    WHERE is_active = true;
+
+COMMENT ON TABLE knowledge.vector_index_snapshot IS
+  'HNSW index rebuild lifecycle for blue/green reindex.';
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000001_semantic_vector_db_seed.sql

@@ -0,0 +1,55 @@
+-- Seed reference data for pg-Semantic-Vector-DB
+-- Source: knowledge/spec/ingestion/corpus_profiles.md, semantic_chunking_spec.md
+
+BEGIN;
+
+INSERT INTO knowledge.corpus_source (book_id, display_name, constraint_tier, locale)
+VALUES
+    ('ado', 'ADO Clinical Textbook', 'loose', 'vi-VN'),
+    ('tny', 'TNY Clinical Textbook', 'loose', 'vi-VN'),
+    ('oho', 'OHO Clinical Textbook', 'section', 'vi-VN'),
+    ('mor', 'MOR Clinical Textbook', 'subsection', 'vi-VN')
+ON CONFLICT (book_id) DO UPDATE
+SET
+    display_name    = EXCLUDED.display_name,
+    constraint_tier = EXCLUDED.constraint_tier,
+    locale          = EXCLUDED.locale;
+
+INSERT INTO knowledge.chunker_profile (
+    chunker_version,
+    description,
+    max_tokens,
+    min_tokens,
+    overlap_tokens,
+    params_json
+)
+VALUES (
+    'header+semantic_v1',
+    'Header pre-split + embedding breakpoint semantic split on oversized pieces',
+    800,
+    120,
+    64,
+    jsonb_build_object(
+        'breakpoint_threshold_type', 'percentile',
+        'breakpoint_threshold_amount', 95,
+        'embedding_model', 'EmbeddingGemma',
+        'protected_blocks', jsonb_build_array('table', 'code_fence', 'short_list')
+    )
+)
+ON CONFLICT (chunker_version) DO UPDATE
+SET
+    description    = EXCLUDED.description,
+    max_tokens     = EXCLUDED.max_tokens,
+    min_tokens     = EXCLUDED.min_tokens,
+    overlap_tokens = EXCLUDED.overlap_tokens,
+    params_json    = EXCLUDED.params_json;
+
+INSERT INTO knowledge.embedding_model (model_name, model_version, dimensions, purpose, is_active)
+VALUES ('EmbeddingGemma', '1', 768, 'rag', true)
+ON CONFLICT (model_name, model_version) DO UPDATE
+SET
+    dimensions = EXCLUDED.dimensions,
+    purpose    = EXCLUDED.purpose,
+    is_active  = EXCLUDED.is_active;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000002_semantic_vector_db_rls_and_rpc.sql

@@ -0,0 +1,119 @@
+-- RLS policies and RAG search RPC for pg-Semantic-Vector-DB on Supabase
+-- Backend ingestion uses service_role (bypasses RLS).
+-- anon/authenticated have no direct access by default.
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Grants (service_role has full access; authenticated denied via RLS)
+-- ---------------------------------------------------------------------------
+GRANT USAGE ON SCHEMA knowledge TO postgres, service_role;
+
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA knowledge TO service_role;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA knowledge TO service_role;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA knowledge
+    GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO service_role;
+
+-- ---------------------------------------------------------------------------
+-- Row Level Security
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_source ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.corpus_edition ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.structure_node ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.logical_unit ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.chunker_profile ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.semantic_chunk ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.chunk_provenance ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.embedding_model ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.semantic_chunk_embedding ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.ingestion_run ENABLE ROW LEVEL SECURITY;
+ALTER TABLE knowledge.vector_index_snapshot ENABLE ROW LEVEL SECURITY;
+
+-- Read-only policy for authenticated backend JWT role (optional future use).
+-- Replace with a dedicated `knowledge_reader` role if you split backend auth.
+DROP POLICY IF EXISTS semantic_chunk_read_active ON knowledge.semantic_chunk;
+CREATE POLICY semantic_chunk_read_active
+    ON knowledge.semantic_chunk
+    FOR SELECT
+    TO authenticated
+    USING (is_active = true);
+
+DROP POLICY IF EXISTS semantic_chunk_embedding_read_indexed ON knowledge.semantic_chunk_embedding;
+CREATE POLICY semantic_chunk_embedding_read_indexed
+    ON knowledge.semantic_chunk_embedding
+    FOR SELECT
+    TO authenticated
+    USING (embedding_status = 'indexed');
+
+-- ---------------------------------------------------------------------------
+-- RAG similarity search RPC
+-- ---------------------------------------------------------------------------
+CREATE OR REPLACE FUNCTION knowledge.match_semantic_chunks(
+    query_embedding public.vector(768),
+    match_count integer DEFAULT 5,
+    filter_book_ids text[] DEFAULT NULL,
+    filter_edition_ids uuid[] DEFAULT NULL
+)
+RETURNS TABLE (
+    chunk_id uuid,
+    content text,
+    parent_title text,
+    page_start integer,
+    page_end integer,
+    section_id integer,
+    subsection_id text,
+    book_id text,
+    edition_id uuid,
+    chunk_index integer,
+    chunker_version text,
+    similarity double precision
+)
+LANGUAGE sql
+STABLE
+SECURITY DEFINER
+SET search_path = knowledge, extensions, public
+AS $$
+    SELECT
+        sc.chunk_id,
+        sc.content,
+        sc.parent_title,
+        sc.page_start,
+        sc.page_end,
+        sc.section_id,
+        sc.subsection_id,
+        cs.book_id,
+        sc.edition_id,
+        sc.chunk_index,
+        sc.chunker_version,
+        1 - (sce.embedding <=> query_embedding) AS similarity
+    FROM knowledge.semantic_chunk_embedding sce
+    INNER JOIN knowledge.semantic_chunk sc
+        ON sc.chunk_id = sce.chunk_id
+    INNER JOIN knowledge.corpus_edition ce
+        ON ce.edition_id = sc.edition_id
+    INNER JOIN knowledge.corpus_source cs
+        ON cs.corpus_source_id = ce.corpus_source_id
+    WHERE ce.status = 'active'
+      AND sc.is_active = true
+      AND sce.embedding_status = 'indexed'
+      AND sce.embedding IS NOT NULL
+      AND (
+          filter_book_ids IS NULL
+          OR cs.book_id = ANY (filter_book_ids)
+      )
+      AND (
+          filter_edition_ids IS NULL
+          OR sc.edition_id = ANY (filter_edition_ids)
+      )
+    ORDER BY sce.embedding <=> query_embedding
+    LIMIT GREATEST(match_count, 1);
+$$;
+
+COMMENT ON FUNCTION knowledge.match_semantic_chunks IS
+  'Top-k cosine similarity search over active-edition semantic chunks. '
+  'Call from backend RAG coordinator with service_role or authenticated JWT.';
+
+GRANT EXECUTE ON FUNCTION knowledge.match_semantic_chunks TO service_role, authenticated;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000003_expose_knowledge_api_schema.sql

@@ -0,0 +1,14 @@
+-- Expose knowledge schema to PostgREST roles used by Supabase Data API.
+-- Hosted projects must also add `knowledge` under Dashboard → Project Settings → API → Exposed schemas.
+
+BEGIN;
+
+GRANT USAGE ON SCHEMA knowledge TO anon, authenticated, service_role;
+GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA knowledge TO service_role;
+GRANT SELECT ON ALL TABLES IN SCHEMA knowledge TO authenticated;
+GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA knowledge TO service_role;
+
+ALTER DEFAULT PRIVILEGES IN SCHEMA knowledge
+    GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO service_role;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000004_chunker_profile_v2.sql

@@ -0,0 +1,35 @@
+-- Register header+semantic_v2 chunker profile (prose/table split before semantic chunking)
+
+BEGIN;
+
+INSERT INTO knowledge.chunker_profile (
+    chunker_version,
+    description,
+    max_tokens,
+    min_tokens,
+    overlap_tokens,
+    params_json
+)
+VALUES (
+    'header+semantic_v2',
+    'Header pre-split + prose/table block extraction + semantic split on oversized prose',
+    800,
+    120,
+    64,
+    jsonb_build_object(
+        'breakpoint_threshold_type', 'percentile',
+        'breakpoint_threshold_amount', 95,
+        'embedding_model', 'EmbeddingGemma',
+        'protected_blocks', jsonb_build_array('table', 'code_fence', 'short_list'),
+        'table_split', true
+    )
+)
+ON CONFLICT (chunker_version) DO UPDATE
+SET
+    description    = EXCLUDED.description,
+    max_tokens     = EXCLUDED.max_tokens,
+    min_tokens     = EXCLUDED.min_tokens,
+    overlap_tokens = EXCLUDED.overlap_tokens,
+    params_json    = EXCLUDED.params_json;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000005_corpus_bibliography.sql

@@ -0,0 +1,139 @@
+-- Bibliographic metadata for corpus_source, corpus_edition, and contributors.
+-- Source: knowledge/corpus/source_metadata.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- Work-level bibliographic fields (corpus_source)
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_source
+    ADD COLUMN IF NOT EXISTS full_title       text,
+    ADD COLUMN IF NOT EXISTS short_title      text,
+    ADD COLUMN IF NOT EXISTS subtitle         text,
+    ADD COLUMN IF NOT EXISTS primary_language text NOT NULL DEFAULT 'vi',
+    ADD COLUMN IF NOT EXISTS subject_area     text,
+    ADD COLUMN IF NOT EXISTS bibliography_json jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+COMMENT ON COLUMN knowledge.corpus_source.full_title IS
+  'Official bibliographic title for citations (may differ from display_name).';
+COMMENT ON COLUMN knowledge.corpus_source.bibliography_json IS
+  'Extensible work-level metadata (resource_type, keywords, series, etc.).';
+
+UPDATE knowledge.corpus_source
+SET full_title = display_name
+WHERE full_title IS NULL;
+
+-- ---------------------------------------------------------------------------
+-- Edition-level publication metadata (corpus_edition)
+-- ---------------------------------------------------------------------------
+ALTER TABLE knowledge.corpus_edition
+    ADD COLUMN IF NOT EXISTS publisher_name    text,
+    ADD COLUMN IF NOT EXISTS publisher_place   text,
+    ADD COLUMN IF NOT EXISTS published_date    date,
+    ADD COLUMN IF NOT EXISTS published_year    smallint,
+    ADD COLUMN IF NOT EXISTS isbn_13           text,
+    ADD COLUMN IF NOT EXISTS isbn_10           text,
+    ADD COLUMN IF NOT EXISTS edition_number    text,
+    ADD COLUMN IF NOT EXISTS volume_label      text,
+    ADD COLUMN IF NOT EXISTS source_uri        text,
+    ADD COLUMN IF NOT EXISTS bibliography_json  jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+COMMENT ON COLUMN knowledge.corpus_edition.published_date IS
+  'Exact publication date when known; otherwise use published_year only.';
+COMMENT ON COLUMN knowledge.corpus_edition.source_uri IS
+  'Canonical source file or URI for this edition (PDF, markdown, etc.).';
+
+CREATE INDEX IF NOT EXISTS idx_corpus_edition_published_year
+    ON knowledge.corpus_edition (published_year);
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_corpus_edition_isbn_13
+    ON knowledge.corpus_edition (isbn_13)
+    WHERE isbn_13 IS NOT NULL;
+
+-- ---------------------------------------------------------------------------
+-- Contributors (authors, editors, translators)
+-- ---------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS knowledge.corpus_contributor (
+    contributor_id     uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    corpus_source_id   uuid NOT NULL
+        REFERENCES knowledge.corpus_source (corpus_source_id)
+        ON DELETE CASCADE,
+    edition_id         uuid
+        REFERENCES knowledge.corpus_edition (edition_id)
+        ON DELETE CASCADE,
+    role               text NOT NULL DEFAULT 'author',
+    full_name          text NOT NULL,
+    name_romanized     text,
+    affiliation        text,
+    orcid              text,
+    sort_order         integer NOT NULL DEFAULT 0,
+    created_at         timestamptz NOT NULL DEFAULT NOW(),
+
+    CONSTRAINT corpus_contributor_role_check
+        CHECK (role IN ('author', 'editor', 'translator', 'compiler', 'institution'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_corpus_contributor_source
+    ON knowledge.corpus_contributor (corpus_source_id, sort_order);
+
+CREATE INDEX IF NOT EXISTS idx_corpus_contributor_edition
+    ON knowledge.corpus_contributor (edition_id, sort_order)
+    WHERE edition_id IS NOT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS uq_corpus_contributor_work_level
+    ON knowledge.corpus_contributor (corpus_source_id, role, full_name)
+    WHERE edition_id IS NULL;
+
+COMMENT ON TABLE knowledge.corpus_contributor IS
+  'People or institutions credited for a corpus work (edition_id NULL) '
+  'or a specific edition.';
+
+-- ---------------------------------------------------------------------------
+-- Citation view for RAG / UI (active edition + aggregated authors)
+-- ---------------------------------------------------------------------------
+CREATE OR REPLACE VIEW knowledge.v_corpus_citation AS
+SELECT
+    cs.corpus_source_id,
+    cs.book_id,
+    cs.display_name,
+    cs.full_title,
+    cs.short_title,
+    cs.subtitle,
+    cs.primary_language,
+    cs.subject_area,
+    cs.constraint_tier,
+    ce.edition_id,
+    ce.edition_label,
+    ce.edition_number,
+    ce.publisher_name,
+    ce.publisher_place,
+    ce.published_date,
+    ce.published_year,
+    ce.isbn_13,
+    ce.isbn_10,
+    ce.source_uri,
+    ce.status AS edition_status,
+    (
+        SELECT string_agg(cc.full_name, '; ' ORDER BY cc.sort_order, cc.full_name)
+        FROM knowledge.corpus_contributor cc
+        WHERE cc.corpus_source_id = cs.corpus_source_id
+          AND cc.role IN ('author', 'editor')
+          AND cc.edition_id IS NULL
+    ) AS contributors
+FROM knowledge.corpus_source cs
+INNER JOIN knowledge.corpus_edition ce
+    ON ce.corpus_source_id = cs.corpus_source_id
+WHERE ce.status = 'active';
+
+COMMENT ON VIEW knowledge.v_corpus_citation IS
+  'Citation bundle for active corpus editions (NFR-18 grounding).';
+
+-- ---------------------------------------------------------------------------
+-- RLS + grants for new table
+-- ---------------------------------------------------------------------------
+GRANT SELECT, INSERT, UPDATE, DELETE ON knowledge.corpus_contributor TO service_role;
+ALTER TABLE knowledge.corpus_contributor ENABLE ROW LEVEL SECURITY;
+
+GRANT SELECT ON knowledge.v_corpus_citation TO service_role, authenticated;
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/migrations/20260705000006_corpus_bibliography_seed.sql

@@ -0,0 +1,368 @@
+-- Seed bibliographic metadata from knowledge/corpus/source_metadata.md
+
+BEGIN;
+
+-- ---------------------------------------------------------------------------
+-- ADO — Dictionary of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Dictionary of Rheumatology',
+    full_title       = 'Dictionary of Rheumatology',
+    short_title      = 'Dictionary of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Musculoskeletal Medicine',
+    bibliography_json = '{
+        "resource_type": "medical_dictionary",
+        "discipline": "rheumatology",
+        "series": null,
+        "keywords": [
+            "rheumatology",
+            "musculoskeletal disorders",
+            "immunology",
+            "osteoporosis",
+            "clinical terminology"
+        ]
+    }'::jsonb
+WHERE book_id = 'ado';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = '2009 English Edition',
+    edition_number    = NULL,
+    publisher_name    = 'Springer-Verlag/Wien',
+    publisher_place   = 'Vienna, Austria',
+    published_year    = 2009,
+    published_date    = NULL,
+    isbn_13           = '9783211685846',
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'ADO.pdf',
+    bibliography_json = '{
+        "library_of_congress_control_number": "2008940520",
+        "imprint": "SpringerWienNewYork",
+        "resource_type": "dictionary"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'ado'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, status,
+    publisher_name, publisher_place, published_year,
+    isbn_13, source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    '2009 English Edition',
+    'active',
+    'Springer-Verlag/Wien',
+    'Vienna, Austria',
+    2009,
+    '9783211685846',
+    'ADO.pdf',
+    '{
+        "library_of_congress_control_number": "2008940520",
+        "imprint": "SpringerWienNewYork",
+        "resource_type": "dictionary"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'ado'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'ado'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('editor', 'Jozef Rovenský', 1),
+        ('editor', 'Juraj Payer', 2),
+        ('author', 'Roy B. Clague', 3),
+        ('author', 'Manfred Herold', 4),
+        ('author', 'Milan Bayer', 5),
+        ('author', 'Helena Tauchmannová', 6),
+        ('author', 'Miroslav Ferenčík', 7),
+        ('author', 'Zdenko Killinger', 8)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'ado';
+
+-- ---------------------------------------------------------------------------
+-- MOR — Manual of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Manual of Rheumatology',
+    full_title       = 'Manual of Rheumatology',
+    short_title      = 'Manual of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Clinical Immunology',
+    bibliography_json = '{
+        "resource_type": "clinical_manual",
+        "discipline": "rheumatology",
+        "keywords": [
+            "rheumatology",
+            "clinical immunology",
+            "arthritis",
+            "musculoskeletal disease",
+            "clinical practice"
+        ]
+    }'::jsonb
+WHERE book_id = 'mor';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'Sixth Edition',
+    edition_number    = '6',
+    publisher_name    = 'CBS Publishers & Distributors Pvt. Ltd.',
+    publisher_place   = 'New Delhi, India',
+    published_year    = 2024,
+    published_date    = NULL,
+    isbn_13           = NULL,
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'MOR.pdf',
+    bibliography_json = '{
+        "format": "ebook",
+        "edition_type": "sixth",
+        "resource_type": "clinical_manual"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'mor'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    publisher_name, publisher_place, published_year,
+    source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'Sixth Edition',
+    '6',
+    'active',
+    'CBS Publishers & Distributors Pvt. Ltd.',
+    'New Delhi, India',
+    2024,
+    'MOR.pdf',
+    '{
+        "format": "ebook",
+        "edition_type": "sixth",
+        "resource_type": "clinical_manual"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'mor'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'mor'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('editor', 'Ramnath Misra', 1),
+        ('editor', 'Sakir Ahmed', 2),
+        ('editor', 'Prasanta Padhan', 3),
+        ('editor', 'Molly Mary Thabah', 4)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'mor';
+
+-- ---------------------------------------------------------------------------
+-- OHO — Oxford Handbook of Rheumatology
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Oxford Handbook of Rheumatology',
+    full_title       = 'Oxford Handbook of Rheumatology',
+    short_title      = 'Oxford Handbook of Rheumatology',
+    subtitle         = NULL,
+    primary_language = 'en',
+    subject_area     = 'Rheumatology; Musculoskeletal Medicine',
+    bibliography_json = '{
+        "resource_type": "handbook",
+        "series": "Oxford Medical Publications",
+        "keywords": [
+            "rheumatology",
+            "musculoskeletal medicine",
+            "clinical handbook",
+            "arthritis",
+            "autoimmune disease"
+        ]
+    }'::jsonb
+WHERE book_id = 'oho';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'Fourth Edition',
+    edition_number    = '4',
+    publisher_name    = 'Oxford University Press',
+    publisher_place   = 'Oxford, United Kingdom',
+    published_year    = 2018,
+    published_date    = NULL,
+    isbn_13           = '9780198728252',
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'OHO.pdf',
+    bibliography_json = '{
+        "series": "Oxford Medical Publications",
+        "library_of_congress_control_number": "2017960672",
+        "ebook_isbn": "9780191043949",
+        "resource_type": "clinical_handbook"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'oho'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    publisher_name, publisher_place, published_year,
+    isbn_13, source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'Fourth Edition',
+    '4',
+    'active',
+    'Oxford University Press',
+    'Oxford, United Kingdom',
+    2018,
+    '9780198728252',
+    'OHO.pdf',
+    '{
+        "series": "Oxford Medical Publications",
+        "library_of_congress_control_number": "2017960672",
+        "ebook_isbn": "9780191043949",
+        "resource_type": "clinical_handbook"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'oho'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'oho'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, v.role, v.full_name, v.sort_order
+FROM knowledge.corpus_source cs
+CROSS JOIN (
+    VALUES
+        ('author', 'Gavin Clunie', 1),
+        ('author', 'Nick Wilkinson', 2),
+        ('author', 'Elena Nikiphorou', 3),
+        ('author', 'Deepak Jadon', 4)
+) AS v(role, full_name, sort_order)
+WHERE cs.book_id = 'oho';
+
+-- ---------------------------------------------------------------------------
+-- TNY — Thuật Ngữ Y Khoa: Cơ Xương Khớp
+-- ---------------------------------------------------------------------------
+UPDATE knowledge.corpus_source
+SET
+    display_name     = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    full_title       = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    short_title      = 'Thuật Ngữ Y Khoa: Cơ Xương Khớp',
+    subtitle         = 'Thuật Ngữ Tiếng Anh Y Khoa Cho Người Mới Bắt Đầu',
+    primary_language = 'vi',
+    subject_area     = 'Musculoskeletal Medicine; Medical Terminology',
+    bibliography_json = '{
+        "resource_type": "educational_material",
+        "series": null,
+        "keywords": [
+            "thuật ngữ y khoa",
+            "cơ xương khớp",
+            "tiếng anh y khoa",
+            "giải phẫu",
+            "bệnh học"
+        ]
+    }'::jsonb
+WHERE book_id = 'tny';
+
+UPDATE knowledge.corpus_edition ce
+SET
+    edition_label     = 'First Edition',
+    edition_number    = '1',
+    publisher_name    = NULL,
+    publisher_place   = NULL,
+    published_year    = NULL,
+    published_date    = NULL,
+    isbn_13           = NULL,
+    isbn_10           = NULL,
+    volume_label      = NULL,
+    source_uri        = 'tny.md',
+    bibliography_json = '{
+        "series": null,
+        "library_of_congress_control_number": null,
+        "ebook_isbn": null,
+        "resource_type": "handout_notes"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE ce.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'tny'
+  AND ce.status = 'active';
+
+INSERT INTO knowledge.corpus_edition (
+    corpus_source_id, edition_label, edition_number, status,
+    source_uri, bibliography_json
+)
+SELECT
+    cs.corpus_source_id,
+    'First Edition',
+    '1',
+    'active',
+    'tny.md',
+    '{
+        "series": null,
+        "library_of_congress_control_number": null,
+        "ebook_isbn": null,
+        "resource_type": "handout_notes"
+    }'::jsonb
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'tny'
+  AND NOT EXISTS (
+      SELECT 1 FROM knowledge.corpus_edition ce
+      WHERE ce.corpus_source_id = cs.corpus_source_id
+        AND ce.status = 'active'
+  );
+
+DELETE FROM knowledge.corpus_contributor cc
+USING knowledge.corpus_source cs
+WHERE cc.corpus_source_id = cs.corpus_source_id
+  AND cs.book_id = 'tny'
+  AND cc.edition_id IS NULL;
+
+INSERT INTO knowledge.corpus_contributor (corpus_source_id, role, full_name, sort_order)
+SELECT cs.corpus_source_id, 'author', 'Nguyễn Thái Duy', 1
+FROM knowledge.corpus_source cs
+WHERE cs.book_id = 'tny';
+
+COMMIT;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/source_query/source_query.sql

@@ -0,0 +1,24 @@
+-- SELECT
+--   sc.chunk_id,
+--   sc.content,
+--   cs.book_id,
+--   cs.display_name AS corpus_display_name,
+--   ce.edition_label,
+--   ce.status AS edition_status,
+--   sc.parent_title,
+--   sc.section_id,
+--   sc.subsection_id,
+--   sc.page_start,
+--   sc.page_end,
+--   sc.chunk_index,
+--   sc.chunker_version
+-- FROM knowledge.semantic_chunk sc
+-- JOIN knowledge.corpus_edition ce
+--   ON ce.edition_id = sc.edition_id
+-- JOIN knowledge.corpus_source cs
+--   ON cs.corpus_source_id = ce.corpus_source_id
+-- WHERE cs.book_id = 'mor' -- other option include: 'oho', 'ado', 'tny'
+--   AND ce.status = 'active'
+--   AND sc.is_active = true
+-- ORDER BY sc.chunk_index
+-- LIMIT 5;

workspace/sprint_1_2/CODEBASE/knowledge/implementation/supabase/upload_supabase.sh

@@ -0,0 +1,15 @@
+PYTHONPATH=. python -m implementation.ingestion.pipeline \
+    --books ado mor oho \
+    --upload-only \
+    --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books ado --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books mor --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env
+
+# PYTHONPATH=. python -m  implementation.ingestion.pipeline --books oho --upload-only  \
+#     --env-path /Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/secrets/aws_secret/.env