update the codebase poc ver1

workspace/sprint_1_2/CODEBASE/knowledge/tests/bounded_pdf.py

@@ -0,0 +1,48 @@
+import fitz  # PyMuPDF
+
+# Open the original dictionary PDF
+input_path = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/tny.pdf"
+output_path = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/tny_bounded.pdf"
+
+doc = fitz.open(input_path)
+
+# Loop through every single page in the document
+for page_num in range(len(doc)):
+    page = doc[page_num]
+    
+    # 1. Dynamically calculate center per page
+    page_width = page.rect.width
+    page_height = page.rect.height
+    center_x = page_width / 2
+
+    # Draw the blue vertical column divider down the middle gutter
+    page.draw_line(
+        fitz.Point(center_x, 0), 
+        fitz.Point(center_x, page_height), 
+        color=(0, 0, 1), 
+        width=3
+    )
+
+    # 2. Get text blocks for this specific page
+    blocks = page.get_text("blocks")
+
+    for b in blocks:
+        x0, y0, x1, y1 = b[0], b[1], b[2], b[3]
+        
+        # OPTIONAL FILTER: Ignore running headers at the very top or footers at the bottom
+        # Adjust these numbers (e.g., 30 and page_height - 30) based on your document margins
+        if y0 < 40 or y1 > (page_height - 40):
+            continue
+            
+        # Draw a green bounding box around the whole paragraph block
+        page.draw_rect(
+            fitz.Rect(x0, y0, x1, y1), 
+            color=(0, 0.6, 0), 
+            width=1.5
+        )
+
+# Save the modifications to a brand new file so we don't overwrite your source corpus
+doc.save(output_path)
+doc.close()
+
+print(f"Saved visual debug map to: {output_path}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/dictionary_text_extract.py

@@ -0,0 +1,38 @@
+import subprocess
+import time
+
+# Define the command as a list of arguments
+command = [
+    "docling", 
+    "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf_chunks/chunk_1-001-005.pdf"
+]
+
+print("Running docling command...")
+
+# Record the start time
+start_time = time.perf_counter()
+
+try:
+    # Run the command and wait for it to complete
+    # text=True and capture_output=True lets you capture the terminal output if needed
+    result = subprocess.run(command, capture_output=True, text=True, check=True)
+    
+    # Record the end time
+    end_time = time.perf_counter()
+    
+    # Calculate total duration
+    execution_time = end_time - start_time
+    
+    print("\n--- Command Executed Successfully ---")
+    print(f"Execution Time: {execution_time:.4f} seconds")
+    
+    # Optional: Print the output from docling if you want to see it
+    # print("\nOutput:")
+    # print(result.stdout)
+
+except subprocess.CalledProcessError as e:
+    end_time = time.perf_counter()
+    print("\n--- Command Failed ---")
+    print(f"Execution Time until failure: {end_time - start_time:.4f} seconds")
+    print(f"Error Code: {e.returncode}")
+    print(f"Error Message:\n{e.stderr}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/extract_embedd_toc.py

@@ -0,0 +1,125 @@
+import json
+import os
+from pypdf import PdfReader
+
+# Configuration
+INPUT_PDF_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/MOR.pdf" # Update to your master textbook PDF path if different
+OUTPUT_JSON_PATH = "toc_structure.json"
+
+# Master Section boundaries provided by you
+SECTION_BOUNDARIES = [
+    {"section": 1, "start": 28, "end": 48, "title": "SECTION I: GENERAL"},
+    {"section": 2, "start": 52, "end": 101, "title": "SECTION II: EXAMINATION"},
+    {"section": 3, "start": 104, "end": 168, "title": "SECTION III: APPROACH TO THE PATIENT"},
+    {"section": 4, "start": 172, "end": 259, "title": "SECTION IV: DRUGS"},
+    {"section": 5, "start": 262, "end": 298, "title": "SECTION V: DISEASE OUTCOME MEASURES"},
+    {"section": 6, "start": 302, "end": 383, "title": "SECTION VI: IMAGING INVESTIGATIONS AND JOINT INJECTIONS"},
+    {"section": 7, "start": 386, "end": 422, "title": "SECTION VII: LABORATORY INVESTIGATIONS"},
+    {"section": 8, "start": 426, "end": 486, "title": "SECTION VIII: RHEUMATOID ARTHRITIS, SPONDYLOARTHRITIS, LUPUS, APS, AND SJÖGREN’S SYNDROME"},
+    {"section": 9, "start": 490, "end": 540, "title": "SECTION IX: SYSTEMIC SCLEROSIS, MYOSITIS, MCTD, AND VASCULITIS"},
+    {"section": 10, "start": 544, "end": 586, "title": "SECTION X: CRYSTAL ARTHRITIDES, SARCOIDOSIS, OSTEOARTHRITIS, OSTEOPOROSIS, JIA, AOSD, MACROPHAGE ACTIVATION SYNDROME, AND SEPTIC ARTHRITIS"},
+    {"section": 11, "start": 590, "end": 635, "title": "SECTION XI: ARTHRITIS IN SYSTEMIC DISEASES AND INFECTIONS, RELAPSING POLYCHONDRITIS, OPTICOSPINAL SYNDROME, IPAF, AND PAH IN CTDs"},
+    {"section": 12, "start": 638, "end": 672, "title": "SECTION XII: KAWASAKI DISEASE, PRIMARY IMMUNODEFICIENCY DISEASES, AUTOINFLAMMATORY SYNDROMES, AMYLOIDOSIS, PANNICULITIS, AND SOFT TISSUE RHEUMATISM"}
+]
+
+def get_section_for_page(page_num):
+    """Finds which section dictionary a specific page number belongs to."""
+    for sec in SECTION_BOUNDARIES:
+        if sec["start"] <= page_num <= sec["end"]:
+            return sec
+    return None
+
+def extract_toc_to_json(pdf_path, output_json):
+    if not os.path.exists(pdf_path):
+        print(f"Error: PDF file not found at {pdf_path}")
+        return
+
+    print("Reading PDF outlines...")
+    reader = PdfReader(pdf_path)
+    
+    try:
+        outline = reader.outline
+        if not outline:
+            print("No embedded outline tree found in this PDF file.")
+            return
+    except Exception as e:
+        print(f"Failed to read outline: {e}")
+        return
+
+    # Flatten out the nested pypdf outline tree down to pure dictionary components
+    raw_elements = []
+    
+    def walk_tree(items):
+        for item in items:
+            if isinstance(item, list):
+                walk_tree(item)
+            else:
+                # FIX: Resolve the IndirectObject pointer into a clean 0-based integer index
+                page_index = reader.get_destination_page_number(item)
+                
+                if page_index is not None:
+                    # Convert 0-indexed page to absolute 1-based PDF page number
+                    page_1_indexed = page_index + 1
+                    raw_elements.append({
+                        "title": item.title.strip(),
+                        "start_page": page_1_indexed
+                    })
+
+    walk_tree(outline)
+    
+    # Initialize JSON output format
+    structured_json = {"sections": []}
+    for sec in SECTION_BOUNDARIES:
+        structured_json["sections"].append({
+            "section_number": sec["section"],
+            "section_title": sec["title"],
+            "page_range": f"{sec['start']}-{sec['end']}",
+            "subsections": []
+        })
+
+    # Filter and map sub-elements to their parent sections
+    subsections_by_section = {i: [] for i in range(1, 13)}
+    
+    for element in raw_elements:
+        parent_sec = get_section_for_page(element["start_page"])
+        if parent_sec:
+            # Avoid re-adding the main section titles if they exist in the outline tree itself
+            if "SECTION" in element["title"].upper() and ("I" in element["title"] or "X" in element["title"] or "V" in element["title"]):
+                continue
+            subsections_by_section[parent_sec["section"]].append(element)
+
+    # Calculate exact page ranges for each sub-section sequentially
+    for sec in structured_json["sections"]:
+        sec_num = sec["section_number"]
+        sec_end_page = SECTION_BOUNDARIES[sec_num - 1]["end"]
+        subs = subsections_by_section[sec_num]
+        
+        # Sort subsections sequentially by their starting page
+        subs = sorted(subs, key=lambda x: x["start_page"])
+        
+        for idx, sub in enumerate(subs):
+            start = sub["start_page"]
+            
+            # The end page of this subsection is 1 page before the next subsection starts,
+            # or the terminal boundary of the section itself.
+            if idx + 1 < len(subs):
+                end = subs[idx + 1]["start_page"] - 1
+                # Corner case handling if multiple subsections sit on the exact same page
+                if end < start:
+                    end = start
+            else:
+                end = sec_end_page
+                
+            sec["subsections"].append({
+                "title": sub["title"],
+                "page_range": f"{start}-{end}"
+            })
+
+    # Export out cleanly formatted JSON data
+    with open(output_json, "w", encoding="utf-8") as json_file:
+        json.dump(structured_json, json_file, indent=4, ensure_ascii=False)
+        
+    print(f"Success! Structural mapping exported to: {output_json}")
+
+if __name__ == "__main__":
+    extract_toc_to_json(INPUT_PDF_PATH, OUTPUT_JSON_PATH)

workspace/sprint_1_2/CODEBASE/knowledge/tests/extract_ner_corpus.py

@@ -0,0 +1,46 @@
+
+from pathlib import Path
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.pipeline.vlm_pipeline import VlmPipeline
+from docling.datamodel.pipeline_options import (
+    VlmPipelineOptions,
+)
+from docling.datamodel import vlm_model_specs
+
+pipeline_options = VlmPipelineOptions(
+    vlm_options=vlm_model_specs.SMOLDOCLING_MLX,  # <-- change the model here
+)
+
+converter = DocumentConverter(
+    format_options={
+        InputFormat.PDF: PdfFormatOption(
+            pipeline_cls=VlmPipeline,
+            pipeline_options=pipeline_options,
+        ),
+    }
+)
+
+script_dir = Path(__file__).resolve().parent.parent
+file_dir = script_dir / "corpus" / "indexs" / "mor_index.pdf"
+output_path = script_dir / "corpus" / "indexs" / "mor_index.md"
+doc = converter.convert(source=file_dir).document
+
+# Extract text content
+text_content = doc.export_to_markdown()
+
+# save to markdown
+output_path.parent.mkdir(parents=True, exist_ok=True)
+with open(output_path, "w", encoding="utf-8") as f:
+    f.write(text_content)
+
+
+
+
+
+
+
+
+
+
+

workspace/sprint_1_2/CODEBASE/knowledge/tests/fast_emb_gemma_install.py

@@ -0,0 +1,45 @@
+import os
+import shutil
+from pathlib import Path
+from fastembed.text.onnx_embedding import OnnxTextEmbedding
+from fastembed.common.model_description import PoolingType
+
+# 1. Define a brand new, completely flat directory outside the messy cache
+BASE_DIR = "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/tests/gemma_emb"
+CLEAN_MODEL_DIR = os.path.join(BASE_DIR, "gemma_final")
+os.makedirs(CLEAN_MODEL_DIR, exist_ok=True)
+
+# 2. Source path where Hugging Face put the files based on your error log
+HF_SNAPSHOT_DIR = os.path.join(BASE_DIR, "models--onnx-community--embeddinggemma-300m-ONNX", "snapshots", "5090578d9565bb06545b4552f76e6bc2c93e4a66")
+
+print("Copying files to a clean, flat directory...")
+files_to_copy = [
+    (os.path.join(HF_SNAPSHOT_DIR, "onnx", "model_fp16.onnx"), "model_fp16.onnx"),
+    (os.path.join(HF_SNAPSHOT_DIR, "onnx", "model_fp16.onnx_data"), "model_fp16.onnx_data"),
+    (os.path.join(HF_SNAPSHOT_DIR, "tokenizer.json"), "tokenizer.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "tokenizer_config.json"), "tokenizer_config.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "special_tokens_map.json"), "special_tokens_map.json"),
+    (os.path.join(HF_SNAPSHOT_DIR, "config.json"), "config.json"),
+]
+
+for src, filename in files_to_copy:
+    dest = os.path.join(CLEAN_MODEL_DIR, filename)
+    if os.path.exists(src) and not os.path.exists(dest):
+        # Using shutil.copy resolves symlinks automatically, copying the real data!
+        shutil.copy(src, dest)
+        print(f"✓ Copied {filename}")
+
+# 3. Initialize directly using the clean, flat folder
+print("\nInitializing Gemma directly via OnnxEmbedding from clean folder...")
+model = OnnxTextEmbedding(
+    model_dir=Path(CLEAN_MODEL_DIR),
+    model_file="model_fp16.onnx", 
+    pooling=PoolingType.MEAN,
+    max_length=512,
+    threads=2
+)
+
+# 4. Verify it works!
+documents = ["Testing local Gemma embedding generation without cache bugs."]
+embeddings = list(model.embed(documents))
+print(f"\n🎉 Success! Generated vector shape: {embeddings[0].shape}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/ingestion_corpus_db.py

@@ -0,0 +1,99 @@
+import asyncio
+import concurrent.futures
+import time
+import aiosqlite
+from docling.document_converter import DocumentConverter
+import re
+# 1. Global Cache: Load docling models ONCE to maximize M1 efficiency
+print("Initializing Docling models into memory...")
+doc_converter = DocumentConverter() 
+print("Models loaded and ready!")
+
+DB_NAME = "tny_ingestion_corpus.db"
+
+TARGET = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/tny/batches"
+
+def get_pdf_files(target_dir):
+    import os
+    pdf_files = []
+    for root, dirs, files in os.walk(target_dir):
+        for file in files:
+            if file.lower().endswith('.pdf'):
+                pdf_files.append(os.path.join(root, file))
+    # Helper function to sort files naturally (e.g., chunk_2 before chunk_10)
+    def natural_sort_key(s):
+        return [int(text) if text.isdigit() else text.lower() for text in re.split(r'(\d+)', s)]
+    return sorted(pdf_files, key=natural_sort_key)
+
+# database initialization
+async def init_db():
+    async with aiosqlite.connect(DB_NAME) as db:
+        await db.execute("""
+            CREATE TABLE IF NOT EXISTS processed_chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_path TEXT,
+                markdown_content TEXT,
+                execution_time REAL,
+                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        await db.commit()
+    print(f"Database '{DB_NAME}' initialized successfully.")
+
+# Heavy CPU/GPU bound worker function (Synchronous context)
+def process_pdf_worker(file_path: str):
+    start = time.perf_counter()
+    
+    # Run the conversion using the pre-loaded global converter
+    result = doc_converter.convert(file_path)
+    markdown_output = result.document.export_to_markdown()
+        
+    end = time.perf_counter()
+    return markdown_output, (end - start)
+
+
+async def save_to_db(file_path: str, markdown_content: str, exec_time: float):
+    print(f"[Database]: Writing results for {file_path.split('/')[-1]} to SQLite...")
+    async with aiosqlite.connect(DB_NAME) as db:
+        await db.execute(
+            "INSERT INTO processed_chunks (source_path, markdown_content, execution_time) VALUES (?, ?, ?)",
+            (file_path, markdown_content, exec_time)
+        )
+        await db.commit()
+    print(f"[Database]: Write complete for {file_path.split('/')[-1]}!")
+
+async def main():
+    await init_db()
+
+    MAX_M1_WORKERS = 2
+    loop = asyncio.get_running_loop()
+    
+    with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_M1_WORKERS) as pool:
+        print("\n--- Starting Ingestion System ---")
+        files_to_ingest = get_pdf_files(TARGET)
+        
+        # 1. Create a helper async function to handle the processing + saving pipeline per file
+        async def pipeline_worker(file_path):
+            # Hand off the heavy Docling job to the pool
+            markdown_content, execution_time = await loop.run_in_executor(pool, process_pdf_worker, file_path)
+            print(f"[Background Worker]: Docling extraction finished for {file_path.split('/')[-1]} in {execution_time:.2f}s")
+            
+            # Save it to the DB as soon as it's done
+            await save_to_db(file_path, markdown_content, execution_time)
+
+        # 2. Fire off ALL tasks into the background immediately (Non-blocking loop)
+        tasks = []
+        for file_path in files_to_ingest:
+            print(f"[Ingest API]: Queueing {file_path.split('/')[-1]}...")
+            tasks.append(pipeline_worker(file_path))
+        
+        # 3. NOW await them all together. This forces the ThreadPoolExecutor 
+        # to actually process 2 files at the exact same time!
+        await asyncio.gather(*tasks)
+
+    print("\n--- All files processed concurrently! ---")
+
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

workspace/sprint_1_2/CODEBASE/knowledge/tests/markdown_chunking_oho.py

@@ -0,0 +1,17 @@
+from langchain_text_splitters import MarkdownHeaderTextSplitter
+from pathlib import Path
+
+oho_markdown_file = Path(__file__).resolve().parent.parent.parent.parent / "CODEBASE" / "knowledge" / "corpus" / "indexs" / "oho_index.md"
+with open(oho_markdown_file, "r", encoding="utf-8") as f:
+    text = f.read()
+
+oho_splitter = MarkdownHeaderTextSplitter(headers_to_split_on = [
+    ("##", "id"), # split based on the alphabetical headers (##) and assign the id to the chunk
+])
+chunks = oho_splitter.split_text(text)
+
+print(f"Total chunks created: {len(chunks)}")
+print(f"First chunk: {chunks[1]}") # start from the 1 to 16
+for i, chunk in enumerate(chunks[1:17], 1):
+    print(f"Chunk {i} : {chunk.page_content}")
+    print("\n \n")

workspace/sprint_1_2/CODEBASE/knowledge/tests/mor_split_page.py

@@ -0,0 +1,96 @@
+import json
+import os
+import re
+import subprocess
+
+# Paths
+JSON_INPUT_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/toc_structure.json"
+MASTER_PDF_PATH = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/mor_bounded.pdf"
+OUTPUT_DIR = "PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/corpus/pdf/mor/sections"
+
+def format_page_number(page_str):
+    """Converts a page string to an integer and pads it to a 3-digit string (e.g., 28 -> '028')."""
+    return f"{int(page_str):03d}"
+
+def clean_filename_part(text):
+    """Extracts section numbers cleanly (e.g., '1.1' from '1.1 History Taking') or replaces bad characters."""
+    # Try to find a subsection number at the start of the title like "1.1" or "10.4"
+    match = re.match(r'^(\d+\.\d+)', text.strip())
+    if match:
+        return match.group(1)
+    
+    # Fallback safe string cleaning if no clean number pattern exists
+    invalid_chars = ['/', '\\', ':', '*', '?', '"', '<', '>', '|', ' ']
+    clean_text = text.strip()
+    for char in invalid_chars:
+        clean_text = clean_text.replace(char, '_')
+    return clean_text[:20]
+
+def split_pdf_by_subsections():
+    # Ensure output folder exists
+    os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+    if not os.path.exists(JSON_INPUT_PATH):
+        print(f"Error: JSON metadata file not found at {JSON_INPUT_PATH}")
+        return
+        
+    with open(JSON_INPUT_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    print("Loaded TOC configurations. Slicing down to SUBSECTION units...\n")
+
+    for sec in data.get("sections", []):
+        sec_num = sec["section_number"]
+        subsections = sec.get("subsections", [])
+        
+        if not subsections:
+            print(f"[Notice] Section {sec_num} has no nested subsections. Skipping.")
+            continue
+
+        print(f"Processing Section {sec_num} -> Found {len(subsections)} subsections to slice:")
+
+        for sub in subsections:
+            title = sub["title"]             # Matches: "1.1 History Taking..."
+            page_range = sub["page_range"]   # Matches: "28-32"
+            
+            try:
+                raw_start, raw_end = page_range.split("-")
+                padded_start = format_page_number(raw_start)
+                padded_end = format_page_number(raw_end)
+            except (ValueError, AttributeError):
+                print(f"  └─ [Warning] Skipping '{title}': Invalid range format '{page_range}'.")
+                continue
+            
+            # Extract a clean identifier prefix like 'sub_1.1' or 'sub_1.2'
+            sub_id = clean_filename_part(title)
+            
+            # Build your targeted filename pattern: sub_1.1_chunk-028-032.pdf
+            output_filename = f"sub_{sub_id}_chunk-{padded_start}-{padded_end}.pdf"
+            output_path = os.path.join(OUTPUT_DIR, output_filename)
+
+            # Formulate the execution payload for qpdf
+            command = [
+                "qpdf",
+                MASTER_PDF_PATH,
+                "--pages",
+                ".",
+                page_range,
+                "--",
+                output_path
+            ]
+
+            try:
+                subprocess.run(command, check=True, capture_output=True, text=True)
+                print(f"  └─ Generated: {output_filename} ({page_range})")
+            except subprocess.CalledProcessError as e:
+                print(f"  └─ [Error] qpdf failed to slice subsection '{title}'.")
+                print(f"     Details: {e.stderr}")
+                continue
+
+    print("\n--- Subsection granularity slicing complete! Check your output folder. ---")
+
+if __name__ == "__main__":
+    if not os.path.exists(MASTER_PDF_PATH):
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'. Please verify your configurations.")
+    else:
+        split_pdf_by_subsections()

workspace/sprint_1_2/CODEBASE/knowledge/tests/oho_split_page.py

@@ -0,0 +1,132 @@
+import re
+import subprocess
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+INDEX_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "index.txt"
+MASTER_PDF_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "oho_bounded.pdf"
+OUTPUT_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "oho" / "sections"
+BATCH_SIZE = 5
+
+SECTION_LINE_PATTERN = re.compile(
+    r"^\s*section\s+(\d+)\s*:\s*(\d+)\s*-\s*(\d+)\s*$",
+    re.IGNORECASE,
+)
+
+
+def format_page_number(page: int) -> str:
+    """Pad a page number to three digits (e.g. 32 -> '032')."""
+    return f"{page:03d}"
+
+
+def parse_index_file(index_path: Path) -> list[dict[str, int]]:
+    """Parse section page ranges from index.txt."""
+    sections: list[dict[str, int]] = []
+
+    with open(index_path, "r", encoding="utf-8") as f:
+        for line_num, line in enumerate(f, start=1):
+            match = SECTION_LINE_PATTERN.match(line)
+            if not match:
+                if line.strip():
+                    print(f"[Warning] Skipping unrecognized line {line_num}: {line.strip()}")
+                continue
+
+            section_num, start_page, end_page = match.groups()
+            sections.append(
+                {
+                    "section_number": int(section_num),
+                    "start_page": int(start_page),
+                    "end_page": int(end_page),
+                }
+            )
+
+    return sections
+
+
+def iter_page_batches(start_page: int, end_page: int, batch_size: int = BATCH_SIZE):
+    """Yield consecutive page batches within a section range."""
+    batch_num = 1
+    current = start_page
+
+    while current <= end_page:
+        batch_end = min(current + batch_size - 1, end_page)
+        yield batch_num, current, batch_end
+        batch_num += 1
+        current = batch_end + 1
+
+
+def run_qpdf_slice(page_range: str, output_path: Path) -> None:
+    command = [
+        "qpdf",
+        str(MASTER_PDF_PATH),
+        "--pages",
+        ".",
+        page_range,
+        "--",
+        str(output_path),
+    ]
+    subprocess.run(command, check=True, capture_output=True, text=True)
+
+
+def split_pdf_by_section_batches() -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not INDEX_PATH.exists():
+        print(f"Error: Index file not found at {INDEX_PATH}")
+        return
+
+    sections = parse_index_file(INDEX_PATH)
+    if not sections:
+        print(f"Error: No valid section entries found in {INDEX_PATH}")
+        return
+
+    print(
+        f"Loaded {len(sections)} section ranges from index.txt. "
+        f"Slicing each section into batches of {BATCH_SIZE} pages...\n"
+    )
+
+    total_batches = 0
+
+    for section in sections:
+        section_num = section["section_number"]
+        start_page = section["start_page"]
+        end_page = section["end_page"]
+        batches = list(iter_page_batches(start_page, end_page))
+
+        print(
+            f"Processing Section {section_num} "
+            f"({start_page}-{end_page}) -> {len(batches)} batch(es):"
+        )
+
+        for batch_num, batch_start, batch_end in batches:
+            page_range = f"{batch_start}-{batch_end}"
+            padded_start = format_page_number(batch_start)
+            padded_end = format_page_number(batch_end)
+            output_filename = (
+                f"sec_{section_num}_batch-{batch_num:02d}_"
+                f"chunk-{padded_start}-{padded_end}.pdf"
+            )
+            output_path = OUTPUT_DIR / output_filename
+
+            try:
+                run_qpdf_slice(page_range, output_path)
+                print(f"  └─ Generated: {output_filename} ({page_range})")
+                total_batches += 1
+            except subprocess.CalledProcessError as e:
+                print(
+                    f"  └─ [Error] qpdf failed to slice section {section_num}, "
+                    f"batch {batch_num}."
+                )
+                print(f"     Details: {e.stderr}")
+
+    print(
+        f"\n--- Section batch slicing complete! "
+        f"Generated {total_batches} PDF(s). Check {OUTPUT_DIR}. ---"
+    )
+
+
+if __name__ == "__main__":
+    if not MASTER_PDF_PATH.exists():
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'.")
+    else:
+        split_pdf_by_section_batches()

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_assemble.py

@@ -0,0 +1,20 @@
+import asyncio
+from pathlib import Path
+from implementation.ingestion.assemble import assemble_logical_units
+from implementation.ingestion.config import resolve_corpus_db_path
+from implementation.ingestion.sqlite_store import load_processed_chunks
+
+async def inspect_assembly(book_id: str = "mor"):
+    db_path = resolve_corpus_db_path(book_id)
+    rows = await load_processed_chunks(db_path)  # list of (id, source_path, markdown_content)
+    units = assemble_logical_units(rows, book_id=book_id)
+    print(f"processed_chunks rows: {len(rows)}")
+    print(f"logical units:         {len(units)}")
+    for unit in units[:10]:
+        print(f"\n--- {unit.logical_unit_id} ---")
+        print(f"pages: {unit.page_start}-{unit.page_end}")
+        print(f"parent_title: {unit.parent_title}")
+        print(f"source_ids: {unit.source_extraction_ids}")
+        print(f"content preview: {unit.content}...")
+
+asyncio.run(inspect_assembly("mor"))

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_bert_ner.py

@@ -0,0 +1,31 @@
+from gliner import GLiNER
+
+model = GLiNER.from_pretrained("gliner-community/gliner_small-v2.5",
+                                map_location="cpu",
+                                cache_dir="/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/tests",
+                                dtype="fp16")
+
+# 1. Define your MSK-domain text snippet (e.g., from a radiology or orthopedic report)
+text = """
+The patient presents with severe lower back pain radiating down the left leg. 
+An MRI of the lumbar spine reveals a moderate L4-L5 disc herniation causing 
+mild stenosis of the neural foramina. We discussed proceeding with a 
+lumbar epidural steroid injection for pain management.
+"""
+
+# 2. Define domain-specific MSK clinical labels
+labels = [
+    "Anatomical_Structure",  # e.g., lumbar spine, L4-L5, neural foramina
+    "Pathology_Condition",   # e.g., disc herniation, stenosis
+    "Symptom",               # e.g., lower back pain
+    "Procedure_Treatment"    # e.g., MRI, lumbar epidural steroid injection
+]
+
+# 3. Predict entities using your model
+entities = model.inference(text, labels, threshold=0.5)
+
+# 4. Print the extracted MSK clinical entities
+print("Extracted MSK Entities:")
+print("-" * 30)
+for entity in entities:
+    print(f"{entity['text']} => {entity['label']}")

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_chunker.py

@@ -0,0 +1,137 @@
+"""Unit tests for prose/table block extraction and chunking."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from implementation.ingestion.assemble import LogicalUnit
+from implementation.ingestion.chunker import (
+    chunk_logical_unit,
+    embedding_text,
+    extract_blocks,
+)
+from implementation.ingestion.config import DEFAULT_CHUNKER_CONFIG
+
+
+@dataclass
+class _MockEmbedder:
+    """Lightweight embedder stub for chunking tests."""
+
+    def count_tokens(self, text: str) -> int:
+        return max(1, len(text.split()))
+
+    def embed_clustering(self, text: str) -> list[float]:
+        return [float(len(text)), 1.0, 0.5]
+
+    def embed_document(self, text: str, *, title: str | None = None) -> list[float]:
+        return [float(len(text)), 1.0, 0.5]
+
+
+def _sample_unit(content: str, book_id: str = "mor") -> LogicalUnit:
+    return LogicalUnit(
+        logical_unit_id="sub_1.1",
+        book_id=book_id,
+        content=content,
+        source_extraction_ids=[1],
+        page_start=1,
+        page_end=2,
+        subsection_id="1.1",
+        parent_title="Sample subsection",
+    )
+
+
+def test_extract_blocks_splits_prose_and_wrapped_table():
+    text = (
+        "Juvenile SLE presents with fever and rash.\n\n"
+        "Table 1. Clinical features\n\n"
+        "| Feature | Frequency |\n"
+        "| --- | --- |\n"
+        "| Fever | High |\n"
+        "Treatment depends on severity."
+    )
+
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 3
+    assert blocks[0].block_type == "prose"
+    assert "Juvenile SLE" in blocks[0].text
+    assert "Table 1." not in blocks[0].text
+    assert blocks[1].block_type == "table"
+    assert blocks[1].text.startswith("Table 1. Clinical features")
+    assert "<table>" in blocks[1].text
+    assert blocks[2].block_type == "prose"
+    assert "Treatment depends" in blocks[2].text
+
+
+def test_extract_blocks_prose_only_when_no_tables():
+    text = "Only prose here with no tabular content."
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 1
+    assert blocks[0].block_type == "prose"
+    assert blocks[0].text == text
+
+
+def test_chunk_logical_unit_emits_separate_table_chunk():
+    unit = _sample_unit(
+        "Intro paragraph about dosing.\n\n"
+        "Table 2. Dosing guide\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>\n\n"
+        "Follow-up monitoring is required."
+    )
+
+    records = chunk_logical_unit(unit, embedder=_MockEmbedder(), config=DEFAULT_CHUNKER_CONFIG)
+
+    assert len(records) == 3
+    assert "Intro paragraph" in records[0].content
+    assert "<table>" in records[1].content
+    assert "Table 2. Dosing guide" in records[1].content
+    assert "Follow-up monitoring" in records[2].content
+    assert records[0].chunk_index == 0
+    assert records[1].chunk_index == 1
+    assert records[2].chunk_index == 2
+    assert records[1].embed_content is not None
+    assert "Sample subsection" not in records[1].embed_content
+    assert "<table>" in records[1].embed_content
+
+
+def test_embedding_text_returns_table_body_without_title():
+    unit = _sample_unit(
+        "Table 2. Dosing guide\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>"
+    )
+    records = chunk_logical_unit(unit, embedder=_MockEmbedder(), config=DEFAULT_CHUNKER_CONFIG)
+    table_record = records[0]
+    table_record = table_record.__class__(**{**table_record.__dict__, "embed_content": None})
+
+    rebuilt = embedding_text(table_record)
+
+    assert "Sample subsection" not in rebuilt
+    assert "<table>" in rebuilt
+
+
+def test_extract_blocks_skips_wrap_when_tables_already_normalized():
+    text = (
+        "Intro paragraph.\n\n"
+        "<table>\n"
+        "| Drug | Dose |\n"
+        "| --- | --- |\n"
+        "| A | 10 mg |\n"
+        "</table>\n\n"
+        "Follow-up text."
+    )
+
+    blocks = extract_blocks(text)
+
+    assert len(blocks) == 3
+    assert blocks[1].block_type == "table"
+    assert blocks[1].text.count("<table>") == 1
+    assert blocks[1].text.count("</table>") == 1

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_embedding.py

@@ -0,0 +1,27 @@
+"""Unit tests for task-routed EmbeddingGemma prefixes."""
+
+from implementation.ingestion.embedding import EmbedTask, format_embed_input
+
+
+def test_retrieval_document_uses_parent_title_in_prefix():
+    result = format_embed_input(
+        "Chunk body text.",
+        EmbedTask.RETRIEVAL_DOCUMENT,
+        title="Juvenile SLE",
+    )
+    assert result == "title: Juvenile SLE | text: Chunk body text."
+
+
+def test_retrieval_document_falls_back_to_none_title():
+    result = format_embed_input("Chunk body.", EmbedTask.RETRIEVAL_DOCUMENT)
+    assert result == "title: none | text: Chunk body."
+
+
+def test_retrieval_query_uses_search_prefix():
+    result = format_embed_input("pediatric dosing", EmbedTask.RETRIEVAL_QUERY)
+    assert result == "task: search result | query: pediatric dosing"
+
+
+def test_clustering_uses_clustering_prefix():
+    result = format_embed_input("Sentence one.", EmbedTask.CLUSTERING)
+    assert result == "task: clustering | query: Sentence one."

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_normalize_newlines.py

@@ -0,0 +1,39 @@
+"""Unit tests for assembled-text newline normalization."""
+
+from implementation.ingestion.assemble import _clean_assembled_text, _normalize_newlines
+
+
+def test_joins_soft_wrapped_paragraph_lines():
+    raw = "He was unable to raise both arms due to\n\nsevere shoulder pain."
+    assert "due to severe shoulder pain" in _normalize_newlines(raw)
+    assert "due to\n\nsevere" not in _normalize_newlines(raw)
+
+
+def test_preserves_markdown_list_lines():
+    raw = "- Spiral fracture : gãy xoắn\n- Communited fracture : gãy nát"
+    normalized = _normalize_newlines(raw)
+    assert normalized.count("\n") == 1
+    assert "- Spiral fracture" in normalized
+    assert "- Communited fracture" in normalized
+
+
+def test_collapses_repeated_spaces():
+    raw = "There  was  no  loss  consciousness"
+    assert _normalize_newlines(raw) == "There was no loss consciousness"
+
+
+def test_collapses_excess_blank_lines():
+    raw = "Line one\n\n\n\nLine two"
+    assert _normalize_newlines(raw) == "Line one\n\nLine two"
+
+
+def test_joins_hyphenation_breaks():
+    raw = "medi-\ncal terminology"
+    assert _normalize_newlines(raw) == "medical terminology"
+
+
+def test_clean_assembled_text_strips_image_markers_and_normalizes():
+    raw = "<!-- image -->\nHe was thrown\n\nfrom his motorcycle."
+    cleaned = _clean_assembled_text(raw)
+    assert "<!-- image -->" not in cleaned
+    assert "thrown from his motorcycle" in cleaned

workspace/sprint_1_2/CODEBASE/knowledge/tests/test_tny_md_processed_chunks.py

@@ -0,0 +1,45 @@
+"""Tests for tny.md → processed_chunks structural rebuild."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from tny_md_processed_chunks import (
+    TNY_MD_PATH,
+    recursive_structure_chunk,
+    validate_source_paths,
+    wrap_tables,
+)
+
+
+def test_wrap_tables_adds_table_tags():
+    sample = "| A | B |\n| --- | --- |\n| 1 | 2 |"
+    wrapped = wrap_tables(sample)
+    assert wrapped.startswith("<table>")
+    assert wrapped.endswith("</table>")
+
+
+def test_tny_md_produces_structural_chunks():
+    if not TNY_MD_PATH.exists():
+        pytest.skip(f"missing fixture: {TNY_MD_PATH}")
+
+    text = TNY_MD_PATH.read_text(encoding="utf-8")
+    chunks = recursive_structure_chunk(text)
+
+    assert len(chunks) >= 10
+    assert any(chunk.block_type == "table" for chunk in chunks)
+    assert all(chunk.markdown_content.strip() for chunk in chunks)
+    validate_source_paths(chunks)
+
+
+def test_tny_md_table_chunk_contains_joint_classification():
+    if not TNY_MD_PATH.exists():
+        pytest.skip(f"missing fixture: {TNY_MD_PATH}")
+
+    chunks = recursive_structure_chunk(TNY_MD_PATH.read_text(encoding="utf-8"))
+    table_chunks = [c for c in chunks if c.block_type == "table"]
+    assert len(table_chunks) == 1
+    assert "Fibrous" in table_chunks[0].markdown_content
+    assert "<table>" in table_chunks[0].markdown_content

workspace/sprint_1_2/CODEBASE/knowledge/tests/tny_md_processed_chunks.py

@@ -0,0 +1,373 @@
+"""Rebuild TNY processed_chunks from corpus/pdf/tny/tny.md.
+
+Pipeline:
+  1. Wrap pipe markdown tables in <table>...</table>
+  2. Recursively split on section (##), subsection (###), sub-subsection (####)
+  3. At each leaf, split prose vs table blocks
+  4. Replace all rows in tny_ingestion_corpus.db processed_chunks
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import re
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+if str(KNOWLEDGE_ROOT) not in sys.path:
+    sys.path.insert(0, str(KNOWLEDGE_ROOT))
+
+import aiosqlite
+
+from implementation.ingestion.assemble import _wrap_markdown_tables
+from implementation.ingestion.chunker import extract_blocks
+from implementation.ingestion.config import resolve_corpus_db_path
+from implementation.ingestion.metadata_parser import parse_source_path
+
+TNY_MD_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "tny.md"
+MD_CHUNKS_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "md_chunks"
+DEBUG_LOG_PATH = (
+    Path(__file__).resolve().parents[6]
+    / ".cursor"
+    / "debug-de2ea8.log"
+)
+
+HEADER_SPLITS: list[tuple[re.Pattern[str], str]] = [
+    (re.compile(r"^(##\s+.+)$", re.MULTILINE), "section"),
+    (re.compile(r"^(###\s+.+)$", re.MULTILINE), "subsection"),
+    (re.compile(r"^(####\s+.+)$", re.MULTILINE), "subsubsection"),
+]
+
+
+@dataclass
+class StructuralChunk:
+    """One processed_chunks row candidate derived from tny.md."""
+
+    markdown_content: str
+    block_type: str
+    section_title: str | None = None
+    subsection_title: str | None = None
+    subsubsection_title: str | None = None
+    ancestry: list[str] = field(default_factory=list)
+
+
+# #region agent log
+def _debug_log(
+    hypothesis_id: str,
+    location: str,
+    message: str,
+    data: dict | None = None,
+    run_id: str = "initial",
+) -> None:
+    payload = {
+        "sessionId": "de2ea8",
+        "runId": run_id,
+        "hypothesisId": hypothesis_id,
+        "location": location,
+        "message": message,
+        "data": data or {},
+        "timestamp": int(time.time() * 1000),
+    }
+    with DEBUG_LOG_PATH.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(payload, ensure_ascii=False) + "\n")
+
+
+# #endregion
+
+
+def wrap_tables(text: str) -> str:
+    """Step 1: wrap pipe markdown tables in <table> tags."""
+    wrapped = _wrap_markdown_tables(text)
+    # #region agent log
+    _debug_log(
+        "H1",
+        "tny_md_processed_chunks.py:wrap_tables",
+        "table wrap complete",
+        {
+            "input_len": len(text),
+            "output_len": len(wrapped),
+            "table_tag_count": wrapped.count("<table>"),
+        },
+    )
+    # #endregion
+    return wrapped
+
+
+def _header_title(line: str) -> str:
+    return re.sub(r"^#{1,6}\s+", "", line.strip())
+
+
+def _split_on_header_level(
+    text: str,
+    level: int,
+    ancestry: list[str],
+) -> list[StructuralChunk]:
+    """Recursively split markdown by section / subsection / sub-subsection."""
+    text = text.strip()
+    if not text:
+        return []
+
+    if level >= len(HEADER_SPLITS):
+        return _split_leaf_blocks(text, ancestry)
+
+    pattern, kind = HEADER_SPLITS[level]
+    matches = list(pattern.finditer(text))
+    if not matches:
+        # #region agent log
+        _debug_log(
+            "H3",
+            "tny_md_processed_chunks.py:_split_on_header_level",
+            "no headers at level; descend",
+            {"level": level, "kind": kind, "text_len": len(text)},
+        )
+        # #endregion
+        return _split_on_header_level(text, level + 1, ancestry)
+
+    chunks: list[StructuralChunk] = []
+    if matches[0].start() > 0:
+        preamble = text[: matches[0].start()].strip()
+        if preamble:
+            chunks.extend(_split_on_header_level(preamble, level + 1, ancestry))
+
+    for index, match in enumerate(matches):
+        start = match.start()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+        section_text = text[start:end].strip()
+        if not section_text:
+            continue
+
+        title = _header_title(match.group(1))
+        next_ancestry = [*ancestry, title]
+        child_chunks = _split_on_header_level(section_text, level + 1, next_ancestry)
+        for chunk in child_chunks:
+            if kind == "section":
+                chunk.section_title = title
+            elif kind == "subsection":
+                chunk.subsection_title = title
+            elif kind == "subsubsection":
+                chunk.subsubsection_title = title
+        chunks.extend(child_chunks)
+
+    return chunks
+
+
+def _split_leaf_blocks(text: str, ancestry: list[str]) -> list[StructuralChunk]:
+    """Split a leaf node into prose and table processed_chunks rows."""
+    blocks = extract_blocks(text)
+    if not blocks:
+        return []
+
+    section_title = ancestry[0] if ancestry else None
+    subsection_title = ancestry[1] if len(ancestry) > 1 else None
+    subsubsection_title = ancestry[2] if len(ancestry) > 2 else None
+
+    chunks: list[StructuralChunk] = []
+    for block in blocks:
+        content = block.text.strip()
+        if not content:
+            continue
+        chunks.append(
+            StructuralChunk(
+                markdown_content=content,
+                block_type=block.block_type,
+                section_title=section_title,
+                subsection_title=subsection_title,
+                subsubsection_title=subsubsection_title,
+                ancestry=list(ancestry),
+            )
+        )
+
+    # #region agent log
+    _debug_log(
+        "H2",
+        "tny_md_processed_chunks.py:_split_leaf_blocks",
+        "leaf blocks extracted",
+        {
+            "ancestry": ancestry,
+            "block_count": len(chunks),
+            "block_types": [c.block_type for c in chunks],
+        },
+    )
+    # #endregion
+    return chunks
+
+
+def recursive_structure_chunk(text: str) -> list[StructuralChunk]:
+    """Step 2: recursive structural split with table isolation."""
+    wrapped = wrap_tables(text)
+    chunks = _split_on_header_level(wrapped, level=0, ancestry=[])
+    non_empty = [chunk for chunk in chunks if chunk.markdown_content.strip()]
+    # #region agent log
+    _debug_log(
+        "H2",
+        "tny_md_processed_chunks.py:recursive_structure_chunk",
+        "recursive split summary",
+        {
+            "total_chunks": len(non_empty),
+            "prose_count": sum(1 for c in non_empty if c.block_type == "prose"),
+            "table_count": sum(1 for c in non_empty if c.block_type == "table"),
+        },
+    )
+    # #endregion
+    return non_empty
+
+
+def _slugify(value: str, max_len: int = 40) -> str:
+    slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
+    return (slug[:max_len] or "chunk").strip("_")
+
+
+def build_source_path(index: int, chunk: StructuralChunk) -> str:
+    """Synthetic batch-style path parseable by metadata_parser for TNY."""
+    MD_CHUNKS_DIR.mkdir(parents=True, exist_ok=True)
+    page = index + 1
+    slug_parts = [_slugify(part) for part in chunk.ancestry if part]
+    slug_parts.append(chunk.block_type)
+    slug = "_".join(slug_parts)[:80] or f"{chunk.block_type}_{page}"
+    filename = f"batch-{page:02d}_chunk-{page:03d}-{page:03d}.pdf"
+    return str(MD_CHUNKS_DIR / filename)
+
+
+def validate_source_paths(chunks: list[StructuralChunk]) -> None:
+    """Ensure every synthetic path parses as TNY batch metadata."""
+    paths = [build_source_path(i, chunk) for i, chunk in enumerate(chunks)]
+    for path in paths:
+        parse_source_path(path, book_id="tny")
+    if len(paths) != len(set(paths)):
+        raise ValueError("Duplicate source_path values generated for processed_chunks")
+
+
+async def rebuild_processed_chunks(
+    db_path: Path,
+    chunks: list[StructuralChunk],
+    *,
+    dry_run: bool = False,
+) -> tuple[int, int]:
+    """Step 3: delete old processed_chunks rows and insert markdown-derived rows."""
+    if not dry_run:
+        async with aiosqlite.connect(db_path) as db:
+            cursor = await db.execute("SELECT COUNT(*) FROM processed_chunks")
+            old_count = int((await cursor.fetchone())[0])
+
+            await db.execute("DELETE FROM processed_chunks")
+            deleted = old_count
+
+            inserted = 0
+            for index, chunk in enumerate(chunks):
+                source_path = build_source_path(index, chunk)
+                await db.execute(
+                    """
+                    INSERT INTO processed_chunks (source_path, markdown_content, execution_time)
+                    VALUES (?, ?, ?)
+                    """,
+                    (source_path, chunk.markdown_content, 0.0),
+                )
+                inserted += 1
+            await db.commit()
+
+            cursor = await db.execute("SELECT COUNT(*) FROM processed_chunks")
+            new_count = int((await cursor.fetchone())[0])
+    else:
+        deleted = -1
+        inserted = len(chunks)
+        new_count = len(chunks)
+
+    # #region agent log
+    _debug_log(
+        "H4",
+        "tny_md_processed_chunks.py:rebuild_processed_chunks",
+        "db rebuild complete",
+        {
+            "dry_run": dry_run,
+            "deleted_old_rows": deleted,
+            "inserted_rows": inserted,
+            "final_row_count": new_count,
+        },
+        run_id="post-fix" if not dry_run else "dry-run",
+    )
+    # #endregion
+    return inserted, new_count
+
+
+async def run(
+    md_path: Path = TNY_MD_PATH,
+    book_id: str = "tny",
+    dry_run: bool = False,
+) -> None:
+    if not md_path.exists():
+        raise FileNotFoundError(f"TNY markdown not found: {md_path}")
+
+    raw_text = md_path.read_text(encoding="utf-8")
+    chunks = recursive_structure_chunk(raw_text)
+    validate_source_paths(chunks)
+
+    db_path = resolve_corpus_db_path(book_id)
+    if not db_path.parent.exists():
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    async with aiosqlite.connect(db_path) as db:
+        await db.execute(
+            """
+            CREATE TABLE IF NOT EXISTS processed_chunks (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                source_path TEXT,
+                markdown_content TEXT,
+                execution_time REAL,
+                processed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+            """
+        )
+        await db.commit()
+
+    inserted, final_count = await rebuild_processed_chunks(db_path, chunks, dry_run=dry_run)
+
+    print(f"Source markdown: {md_path}")
+    print(f"Structural chunks: {len(chunks)} (prose={sum(1 for c in chunks if c.block_type == 'prose')}, "
+          f"table={sum(1 for c in chunks if c.block_type == 'table')})")
+    print(f"Database: {db_path}")
+    if dry_run:
+        print("Dry run — no database writes performed.")
+    else:
+        print(f"Replaced processed_chunks: inserted={inserted}, final_count={final_count}")
+
+    for index, chunk in enumerate(chunks[:5], start=1):
+        print(f"\n--- preview {index}/{len(chunks)} [{chunk.block_type}] ---")
+        print(f"ancestry: {' > '.join(chunk.ancestry) or '(root)'}")
+        print(chunk.markdown_content[:200] + ("..." if len(chunk.markdown_content) > 200 else ""))
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Rebuild TNY processed_chunks from tny.md with table wrap + structural split.",
+    )
+    parser.add_argument(
+        "--md-path",
+        type=Path,
+        default=TNY_MD_PATH,
+        help="Path to tny.md (default: corpus/pdf/tny/tny.md)",
+    )
+    parser.add_argument(
+        "--book-id",
+        default="tny",
+        help="Book id for checkpoint DB resolution (default: tny)",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Split and validate only; do not write SQLite.",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    asyncio.run(run(md_path=args.md_path, book_id=args.book_id, dry_run=args.dry_run))
+
+
+if __name__ == "__main__":
+    main()

workspace/sprint_1_2/CODEBASE/knowledge/tests/tny_split_page.py

@@ -0,0 +1,88 @@
+import subprocess
+from pathlib import Path
+
+KNOWLEDGE_ROOT = Path(__file__).resolve().parent.parent
+MASTER_PDF_PATH = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "tny_bounded.pdf"
+OUTPUT_DIR = KNOWLEDGE_ROOT / "corpus" / "pdf" / "tny" / "batches"
+BATCH_SIZE = 5
+
+
+def format_page_number(page: int) -> str:
+    """Pad a page number to three digits (e.g. 1 -> '001')."""
+    return f"{page:03d}"
+
+
+def get_pdf_page_count(pdf_path: Path) -> int:
+    result = subprocess.run(
+        ["qpdf", "--show-npages", str(pdf_path)],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return int(result.stdout.strip())
+
+
+def iter_page_batches(start_page: int, end_page: int, batch_size: int = BATCH_SIZE):
+    """Yield consecutive page batches across the full document."""
+    batch_num = 1
+    current = start_page
+
+    while current <= end_page:
+        batch_end = min(current + batch_size - 1, end_page)
+        yield batch_num, current, batch_end
+        batch_num += 1
+        current = batch_end + 1
+
+
+def run_qpdf_slice(page_range: str, output_path: Path) -> None:
+    command = [
+        "qpdf",
+        str(MASTER_PDF_PATH),
+        "--pages",
+        ".",
+        page_range,
+        "--",
+        str(output_path),
+    ]
+    subprocess.run(command, check=True, capture_output=True, text=True)
+
+
+def split_pdf_into_batches() -> None:
+    OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    total_pages = get_pdf_page_count(MASTER_PDF_PATH)
+    batches = list(iter_page_batches(1, total_pages))
+
+    print(
+        f"Loaded '{MASTER_PDF_PATH.name}' ({total_pages} pages). "
+        f"Slicing into batches of {BATCH_SIZE} pages...\n"
+    )
+
+    total_generated = 0
+
+    for batch_num, batch_start, batch_end in batches:
+        page_range = f"{batch_start}-{batch_end}"
+        padded_start = format_page_number(batch_start)
+        padded_end = format_page_number(batch_end)
+        output_filename = f"batch-{batch_num:02d}_chunk-{padded_start}-{padded_end}.pdf"
+        output_path = OUTPUT_DIR / output_filename
+
+        try:
+            run_qpdf_slice(page_range, output_path)
+            print(f"  └─ Generated: {output_filename} ({page_range})")
+            total_generated += 1
+        except subprocess.CalledProcessError as e:
+            print(f"  └─ [Error] qpdf failed to slice batch {batch_num}.")
+            print(f"     Details: {e.stderr}")
+
+    print(
+        f"\n--- Batch slicing complete! "
+        f"Generated {total_generated} PDF(s). Check {OUTPUT_DIR}. ---"
+    )
+
+
+if __name__ == "__main__":
+    if not MASTER_PDF_PATH.exists():
+        print(f"Error: Master PDF not found at '{MASTER_PDF_PATH}'.")
+    else:
+        split_pdf_into_batches()

workspace/sprint_1_2/CODEBASE/knowledge/tests/use_gemma.py

@@ -0,0 +1,83 @@
+import torch
+import numpy as np
+from transformers import AutoTokenizer
+from optimum.onnxruntime import ORTModelForFeatureExtraction
+
+LOCAL_MODEL_DIR = "/Users/davestran/Downloads/vkist_internship/PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/knowledge/implementation/model/gemma_final"
+
+if not hasattr(torch, "int4"):
+    torch.int4 = torch.int8
+
+tokenizer = AutoTokenizer.from_pretrained(LOCAL_MODEL_DIR, local_files_only=True)
+model = ORTModelForFeatureExtraction.from_pretrained(
+    LOCAL_MODEL_DIR, file_name="model_fp16.onnx", local_files_only=True, provider="CPUExecutionProvider", use_io_binding=False
+)
+
+def get_embedding(text, target_dim=256):
+    inputs = tokenizer(text, max_length=512, padding=True, truncation=True, return_tensors="pt")
+    with torch.no_grad():
+        outputs = model(**inputs)
+    raw_states = outputs.last_hidden_state.numpy()
+    mean_pooled = np.mean(raw_states, axis=1)[0]
+    scaled = mean_pooled[:target_dim]
+    return scaled / np.linalg.norm(scaled)
+
+def cosine_similarity(v1, v2):
+    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
+
+# --- TEST DATA ---
+# query_raw = "How do I fix a leaking faucet pipe?"
+# doc_relevant_raw = "To repair a copper pipe valve joint, first isolate the main water supply and drain the lines."
+# doc_irrelevant_raw = "The company announced a 3-for-1 stock split starting next Monday for retail investors."
+
+
+# --- THE HARD NEGATIVE DATA ---
+query_raw = "Apple stock price performance this quarter"
+doc_relevant_raw = "AAPL shares rallied 3% following strong fiscal earnings reports today."
+doc_irrelevant_raw = "The organic Honeycrisp apple crop harvested this quarter yielded excellent crisp fruit."
+
+# 1. UNTUNED TEST (No instructions)
+print("--- RUNNING UNTUNED TEST ---")
+v_query_ut = get_embedding(query_raw)
+v_rel_ut = get_embedding(doc_relevant_raw)
+v_irrel_ut = get_embedding(doc_irrelevant_raw)
+
+sim_rel_ut = cosine_similarity(v_query_ut, v_rel_ut)
+sim_irrel_ut = cosine_similarity(v_query_ut, v_irrel_ut)
+print(f"Untuned Cosine Similarity (Relevant Doc):   {sim_rel_ut:.4f}")
+print(f"Untuned Cosine Similarity (Irrelevant Doc): {sim_irrel_ut:.4f}")
+print(f"Untuned Margin Gap:                         {sim_rel_ut - sim_irrel_ut:.4f}\n")
+
+# --- CORRECTED TUNED TEST (Google Asymmetric Prompt Routing) ---
+print("--- RUNNING CORRECTED TUNED TEST ---")
+
+AVAILABLE_TASK = {
+    "query": "task: search result | query: ",
+    "document": "title: none | text: ",
+    "BitextMining": "task: search result | query: ",
+    "Clustering": "task: clustering | query: ",
+    "Classification": "task: classification | query: ",
+    "InstructionRetrieval": "task: code retrieval | query: ",
+    "MultilabelClassification": "task: classification | query: ",
+    "PairClassification": "task: sentence similarity | query: ",
+    "Reranking": "task: search result | query: ",
+    "Retrieval": "task: search result | query: ",
+    "Retrieval-query": "task: search result | query: ",
+    "Retrieval-document": "title: none | text: "
+}
+# 1. Define distinct query vs document wrappers
+QUERY_PREFIX = AVAILABLE_TASK["query"]
+DOC_PREFIX = AVAILABLE_TASK["Retrieval-document"]
+
+# 2. Apply them separately (No overlapping text injected across documents!)
+v_query_t = get_embedding(QUERY_PREFIX + query_raw)
+v_rel_t = get_embedding(DOC_PREFIX + doc_relevant_raw)
+v_irrel_t = get_embedding(DOC_PREFIX + doc_irrelevant_raw)
+
+# 3. Calculate similarity
+sim_rel_t = cosine_similarity(v_query_t, v_rel_t)
+sim_irrel_t = cosine_similarity(v_query_t, v_irrel_t)
+
+print(f"Tuned Cosine Similarity (Relevant Doc):     {sim_rel_t:.4f}")
+print(f"Tuned Cosine Similarity (Irrelevant Doc):   {sim_irrel_t:.4f}")
+print(f"Tuned Margin Gap:                           {sim_rel_t - sim_irrel_t:.4f}")