update the cv modal inference proxy server with optimization

workspace/sprint_1_2/CODEBASE/backend/cv_inference_server.py

@@ -31,7 +31,7 @@ os.environ.setdefault("TRITON_ENDPOINT", DEFAULT_TRITON_ENDPOINT)
 import uvicorn
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
-
+import asyncio
 from backend.routers import cv_inference
 
 logger = logging.getLogger(__name__)
@@ -44,10 +44,7 @@ async def lifespan(app: FastAPI):
     logger.info("Starting CV inference service on Triton: %s", os.getenv("TRITON_ENDPOINT"))
     from backend.services.triton_warmup import warmup_triton_models
 
-    try:
-        await warmup_triton_models()
-    except Exception as exc:
-        logger.warning("Triton warmup failed — first clinical request may be slow: %s", exc)
+    warmup_task = asyncio.create_task(warmup_triton_models())
     yield
     logger.info("Shutting down CV inference service")
 

workspace/sprint_1_2/CODEBASE/backend/implementation/adapters/triton_adapter.py

@@ -3,16 +3,34 @@ import json
 from typing import Any
 import numpy as np
 import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
 
 
 class TritonAdapter:
     def __init__(self, endpoint_url: str, timeout: float = 60.0):
         self.endpoint_url = endpoint_url.rstrip("/")
         self.timeout = timeout
+        self._session = self._build_session()
 
+    @staticmethod
+    def _build_session() -> requests.Session:
+        session = requests.Session()
+        retry = Retry(
+            total=0,
+            connect=0,
+            read=0,
+            redirect=0,
+            status=0,
+            raise_on_status=False,
+        )
+        adapter = HTTPAdapter(max_retries=retry, pool_connections=20, pool_maxsize=50)
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+        return session
 
     async def close(self):
-        pass
+        await asyncio.to_thread(self._session.close)
 
 
     async def infer(
@@ -61,7 +79,7 @@ class TritonAdapter:
         }
 
         url = f"{self.endpoint_url}/v2/models/{model_name}/infer"
-        response = requests.post(url, data=body, headers=headers, timeout=self.timeout)
+        response = self._session.post(url, data=body, headers=headers, timeout=self.timeout)
         response.raise_for_status()
         return self._parse_binary_response(response.headers, response.content)
 
@@ -91,7 +109,7 @@ class TritonAdapter:
 
     def _model_ready_sync(self, model_name: str) -> bool:
         url = f"{self.endpoint_url}/v2/models/{model_name}"
-        response = requests.get(url, timeout=self.timeout)
+        response = self._session.get(url, timeout=self.timeout)
         if response.status_code == 404:
             return False
         response.raise_for_status()
@@ -113,7 +131,7 @@ class TritonAdapter:
         url = f"{self.endpoint_url}/v2/repository/index"
         
         # 2. Change requests.get to requests.post with an empty json payload {}
-        response = requests.post(url, json={}, timeout=self.timeout)
+        response = self._session.post(url, json={}, timeout=self.timeout)
         response.raise_for_status()
         
         data = response.json()

workspace/sprint_1_2/CODEBASE/backend/implementation/tasks/cv_tasks.py

@@ -0,0 +1,50 @@
+import asyncio
+import base64
+import io
+import logging
+from typing import Any
+
+from PIL import Image
+
+from backend.implementation.postprocessing.calibration import calibration_config_from_params
+from backend.services.cv_inference_service import CvInferenceOptions, run_single
+from backend.services.celery_app import celery_app
+
+logger = logging.getLogger(__name__)
+
+
+def _decode_base64_to_pil(b64_str: str) -> Image.Image:
+    raw = base64.b64decode(b64_str)
+    return Image.open(io.BytesIO(raw)).convert("RGB")
+
+
+@celery_app.task(bind=True, name="cv_inference.run_chunk", max_retries=2, default_retry_delay=10)
+def run_cv_chunk(self, chunk_payload: dict) -> list[dict]:
+    images_b64 = chunk_payload.get("images_b64", [])
+    frame_ids = chunk_payload.get("frame_ids", [])
+    calibration_params = chunk_payload.get("calibration", {})
+    model_versions = chunk_payload.get("model_versions")
+
+    if not images_b64:
+        return []
+
+    images = [_decode_base64_to_pil(b64) for b64 in images_b64]
+    calibration = calibration_config_from_params(calibration_params)
+    options = CvInferenceOptions(
+        calibration=calibration,
+        model_versions=model_versions,
+        use_cache=False,
+    )
+
+    async def _run():
+        return await asyncio.gather(*[
+            run_single(img, frame_id=fid, options=options)
+            for img, fid in zip(images, frame_ids)
+        ])
+
+    try:
+        results = asyncio.run(_run())
+        return results
+    except Exception as exc:
+        logger.exception("Celery chunk task failed: %s", exc)
+        raise self.retry(exc=exc, countdown=10, max_retries=2)

workspace/sprint_1_2/CODEBASE/backend/logging/logging_config.py

@@ -0,0 +1,29 @@
+import logging
+import sys
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+
+LOG_DIR = Path("logs")
+LOG_DIR.mkdir(exist_ok=True)
+LOG_FORMAT = "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s"
+LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+def setup_logging():
+    root = logging.getLogger()
+    root.setLevel(logging.DEBUG)
+    root.handlers.clear()
+    formatter = logging.Formatter(LOG_FORMAT, datefmt=LOG_DATE_FORMAT)
+    console = logging.StreamHandler(sys.stdout)
+    console.setLevel(logging.INFO)
+    console.setFormatter(formatter)
+    root.addHandler(console)
+    file_handler = RotatingFileHandler(
+        LOG_DIR / "app.log",
+        maxBytes=10 * 1024 * 1024,
+        backupCount=5,
+        encoding="utf-8",
+    )
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(formatter)
+    root.addHandler(file_handler)
+    logging.getLogger("httpx").setLevel(logging.WARNING)
+    logging.getLogger("uvicorn.access").setLevel(logging.INFO)

workspace/sprint_1_2/CODEBASE/backend/routers/cv_inference.py

@@ -18,6 +18,10 @@ from backend.implementation.triton_batch import TRITON_MAX_BATCH_SIZE
 from backend.services import cv_result_cache
 from backend.services import triton_runtime_service as triton_runtime
 from backend.services.cv_inference_service import CvBatchResult, CvInferenceOptions, run_batch, run_single
+from backend.services import cv_celery_service
+from backend.logging.logging_config import setup_logging
+
+setup_logging()
 
 logger = logging.getLogger(__name__)
 
@@ -135,14 +139,15 @@ async def cv_inference_health():
         )
 
 
-@router.post("/analyze")
+@router.post("/analyze") # deprecated
 async def analyze_upload(
     image: UploadFile = File(...),
     calibration: str | None = Form(default=None),
 ):
     """Spec-compliant CV pipeline: CLAHE → angle → inflammation → conditional segmentation."""
     image_pil = await _load_upload_image(image)
-    options = _build_options(_parse_calibration_form(calibration), use_cache=False)
+    
+    options = _build_options(_parse_calibration_form(calibration), use_cache=True)
 
     try:
         result = await run_single(image_pil, frame_id=None, options=options)
@@ -168,11 +173,13 @@ async def analyze_batch_upload(
     calibration: str | None = Form(default=None),
 ):
     """Spec-compliant CV batch — one full pipeline per frame (angle-first, gated segmentation)."""
+    logger.info("Starting analyze batch upload")
     if not images:
         raise HTTPException(status_code=400, detail="At least one image is required")
 
     try:
         id_list = json.loads(frame_ids)
+        # logger.info("Start to check the id_list")
     except json.JSONDecodeError as exc:
         raise HTTPException(status_code=400, detail="frame_ids must be a JSON array of strings") from exc
 
@@ -213,6 +220,68 @@ async def analyze_batch_upload(
         raise HTTPException(status_code=500, detail=str(exc)) from exc
 
 
+@router.post("/analyze/batch/celery")
+async def analyze_batch_celery(
+    images: list[UploadFile] = File(...),
+    frame_ids: str = Form(...),
+    calibration: str | None = Form(default=None),
+):
+    """Experiment: async chunk fan-out via Celery + Redis."""
+    if not images:
+        raise HTTPException(status_code=400, detail="At least one image is required")
+
+    try:
+        id_list = json.loads(frame_ids)
+    except json.JSONDecodeError as exc:
+        raise HTTPException(status_code=400, detail="frame_ids must be a JSON array of strings") from exc
+
+    if not isinstance(id_list, list) or not all(isinstance(x, str) for x in id_list):
+        raise HTTPException(status_code=400, detail="frame_ids must be a JSON array of strings")
+    if len(id_list) != len(images):
+        raise HTTPException(status_code=400, detail="frame_ids length must match images count")
+
+    image_pils: list[Image.Image] = []
+    for upload in images:
+        image_pils.append(await _load_upload_image(upload))
+
+    options = _build_options(_parse_calibration_form(calibration))
+
+    try:
+        job_id = cv_celery_service.submit_celery_batch(image_pils, id_list, options)
+        return JSONResponse({
+            "success": True,
+            "job_id": job_id,
+            "image_count": len(image_pils),
+            "mode": "celery-chunk-fanout",
+            "chunk_size": cv_celery_service.CELERY_CHUNK_SIZE,
+        })
+    except Exception as exc:
+        logger.exception("Celery batch submit failed")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+@router.get("/analyze/batch/celery/{job_id}")
+async def analyze_batch_celery_result(job_id: str):
+    """Poll result for a Celery chunk-fan-out batch job."""
+    try:
+        result = await cv_celery_service.get_celery_batch_result(job_id)
+        status = result.get("status")
+        if status == "pending":
+            return JSONResponse(
+                status_code=202,
+                content=result,
+            )
+        if status == "unknown":
+            return JSONResponse(
+                status_code=404,
+                content=result,
+            )
+        return JSONResponse(result)
+    except Exception as exc:
+        logger.exception("Celery batch result fetch failed")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
 @router.post("/segment")
 @router.post("/segment/batch")
 @router.post("/angle")

workspace/sprint_1_2/CODEBASE/backend/routers/run_cv_inference.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Launch the standalone CV inference FastAPI server (Modal Triton).
+#
+# Usage:
+#   ./backend/run_cv_inference.sh
+#
+# Works from any directory: it resolves the CODEBASE root relative to this
+# script, sets PYTHONPATH so `import backend...` resolves, then runs the
+# server as a module.
+#
+# Override defaults via env vars, e.g.:
+#   CV_INFERENCE_PORT=8080 ./backend/run_cv_inference.sh
+#
+set -euo pipefail
+
+# CODEBASE root = grandparent dir of this script's directory (script lives in backend/routers/).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CODEBASE_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+cd "${CODEBASE_ROOT}"
+
+export PYTHONPATH="${CODEBASE_ROOT}:${PYTHONPATH:-}"
+
+# exec python -m backend.cv_inference_server
+
+exec uvicorn backend.cv_inference_server:app --host 0.0.0.0 --port ${CV_INFERENCE_PORT:-8001}

workspace/sprint_1_2/CODEBASE/backend/services/celery_app.py

@@ -0,0 +1,22 @@
+from celery import Celery
+
+from backend.implementation import config
+
+celery_app = Celery(
+    "cv-inference",
+    broker=f"redis://{config.REDIS_HOST}:{config.REDIS_PORT}/{config.REDIS_DB}",
+    backend=f"redis://{config.REDIS_HOST}:{config.REDIS_PORT}/{config.REDIS_DB}",
+    # Explicit include — autodiscover looks for tasks.py, not cv_tasks.py
+    include=["backend.implementation.tasks.cv_tasks"],
+)
+
+# Must match the @task(name=...) value, not the Python module path
+celery_app.conf.task_routes = {
+    "cv_inference.run_chunk": {"queue": "cv-inference"},
+}
+
+celery_app.conf.task_serializer = "json"
+celery_app.conf.result_serializer = "json"
+celery_app.conf.accept_content = ["json"]
+celery_app.conf.result_expires = 3600
+celery_app.conf.task_default_queue = "cv-inference"

workspace/sprint_1_2/CODEBASE/backend/services/cv_celery_service.py

@@ -0,0 +1,130 @@
+import base64
+import dataclasses
+import logging
+import os
+import time
+from typing import Any
+
+from backend.services.cv_inference_service import CvInferenceOptions, _encode_image_to_bytes
+from backend.implementation.tasks.cv_tasks import run_cv_chunk
+from backend.services.celery_app import celery_app
+
+logger = logging.getLogger(__name__)
+
+CELERY_CHUNK_SIZE = int(os.getenv("CELERY_CHUNK_SIZE", "4"))
+CELERY_BATCH_POLL_CACHE_TTL_MS = float(os.getenv("CELERY_BATCH_POLL_CACHE_TTL_MS", "2000"))
+
+_batch_poll_cache: dict[str, tuple[float, dict[str, Any]]] = {}
+_batch_timings: dict[str, float] = {}
+
+
+def submit_celery_batch(images, frame_ids, options):
+    if not images:
+        raise ValueError("images must not be empty")
+    if len(frame_ids) != len(images):
+        raise ValueError("frame_ids length must match images length")
+
+    from celery import group
+
+    chunks = []
+    for i in range(0, len(images), CELERY_CHUNK_SIZE):
+        chunk_imgs = images[i : i + CELERY_CHUNK_SIZE]
+        chunk_fids = frame_ids[i : i + CELERY_CHUNK_SIZE]
+        b64_images = [
+            base64.b64encode(_encode_image_to_bytes(img)).decode() for img in chunk_imgs
+        ]
+        chunk_payload = {
+            "images_b64": b64_images,
+            "frame_ids": chunk_fids,
+            "calibration": dataclasses.asdict(options.calibration) if options.calibration else {},
+            "model_versions": options.model_versions,
+        }
+        chunks.append(chunk_payload)
+
+    job = group(run_cv_chunk.s(chunk) for chunk in chunks)
+    result = job.apply_async()
+    try:
+        result.save()
+    except Exception:
+        logger.exception("Failed to persist Celery GroupResult for job %s", result.id)
+
+    _batch_timings[result.id] = time.monotonic()
+    logger.info(
+        "Celery batch submitted job_id=%s chunks=%d images=%d",
+        result.id,
+        len(chunks),
+        len(images),
+    )
+    return result.id
+
+
+async def get_celery_batch_result(job_id):
+    from celery.result import GroupResult
+
+    now = time.monotonic()
+    cached = _batch_poll_cache.get(job_id)
+    if cached and cached[0] > now:
+        return cached[1]
+
+    result = GroupResult.restore(job_id, app=celery_app)
+    if result is None:
+        payload = {
+            "status": "unknown",
+            "detail": "Job ID not found in result backend",
+        }
+        _batch_poll_cache[job_id] = (now + CELERY_BATCH_POLL_CACHE_TTL_MS / 1000, payload)
+        return payload
+
+    completed = result.completed_count()
+    total = len(result.results)
+
+    if not result.ready():
+        payload = {
+            "status": "pending",
+            "completed": completed,
+            "total": total,
+            "progress": round(completed / total, 2) if total > 0 else 0,
+        }
+        _batch_poll_cache[job_id] = (now + CELERY_BATCH_POLL_CACHE_TTL_MS / 1000, payload)
+        return payload
+
+    if result.failed():
+        errors = []
+        for r in result.results:
+            if r.failed():
+                errors.append(str(r.result))
+        payload = {
+            "status": "failed",
+            "errors": errors,
+        }
+        _batch_poll_cache[job_id] = (now + CELERY_BATCH_POLL_CACHE_TTL_MS / 1000, payload)
+        _log_batch_timing(job_id, "failed")
+        return payload
+
+    all_results = []
+    for r in result.results:
+        chunk_results = r.get()
+        all_results.extend(chunk_results)
+
+    payload = {
+        "status": "completed",
+        "image_count": len(all_results),
+        "results": all_results,
+    }
+    _batch_poll_cache[job_id] = (now + CELERY_BATCH_POLL_CACHE_TTL_MS / 1000, payload)
+    _log_batch_timing(job_id, "completed")
+    return payload
+
+
+def _log_batch_timing(job_id: str, status: str) -> None:
+    start = _batch_timings.pop(job_id, None)
+    if start is not None:
+        duration_ms = (time.monotonic() - start) * 1000
+        logger.info(
+            "Celery batch %s job_id=%s duration_ms=%.0f",
+            status,
+            job_id,
+            duration_ms,
+        )
+    else:
+        logger.info("Celery batch %s job_id=%s (no start time recorded)", status, job_id)

workspace/sprint_1_2/CODEBASE/backend/services/cv_inference_service.py

@@ -5,6 +5,7 @@ import asyncio
 import base64
 import io
 import logging
+import os
 from dataclasses import dataclass
 from typing import Any
 
@@ -52,8 +53,6 @@ SEGMENT_CLASSES_POST = {
     6: "baker's cyst",
 }
 
-_triton_pipeline_lock = asyncio.Lock()
-
 
 @dataclass
 class CvInferenceOptions:
@@ -76,6 +75,12 @@ def _encode_image_to_data_url(image_pil: Image.Image) -> str:
     return f"data:image/png;base64,{encoded}"
 
 
+def _encode_image_to_bytes(image_pil: Image.Image) -> bytes:
+    buffered = io.BytesIO()
+    image_pil.save(buffered, format="PNG")
+    return buffered.getvalue()
+
+
 def _build_inflammation_payload(logits_row: np.ndarray, config: CalibrationConfig) -> dict:
     interpreted = interpret_inflammation_logits(logits_row, config)
     return {
@@ -147,6 +152,9 @@ def _build_segmentation_result(
     classes_detected = [name for name, mask in masks.items() if np.sum(mask) > 0 and name != "background"]
     color_legend = _build_color_legend(classes_detected, angle_type)
 
+    segmented_png_bytes = _encode_image_to_bytes(overlay)
+    segmented_b64 = base64.b64encode(segmented_png_bytes).decode()
+
     result: dict[str, Any] = {
         "success": True,
         "angle": angle_payload,
@@ -161,7 +169,7 @@ def _build_segmentation_result(
         },
         "images": {
             "enhanced": enhanced_data_url,
-            "segmented": _encode_image_to_data_url(overlay),
+            "segmented": f"data:image/png;base64,{segmented_b64}",
         },
         "models_used": {
             "angle": angle_model,
@@ -281,19 +289,30 @@ async def _run_batch_uncached(
     if len(frame_ids) != len(images):
         raise ValueError("frame_ids length must match images length")
 
-    async with _triton_pipeline_lock:
-        results: list[dict[str, Any]] = []
-        infer_modes: list[str] = []
-        triton_call_count = 0
-        for image_pil, fid in zip(images, frame_ids, strict=True):
-            item, mode, calls = await _run_spec_cv_pipeline_single(
+    concurrency = int(os.getenv("CV_BATCH_CONCURRENCY", "2"))
+    semaphore = asyncio.Semaphore(concurrency)
+
+    async def process_one(image_pil: Image.Image, fid: str, index: int):
+        async with semaphore:
+            if index > 0:
+                await asyncio.sleep(min(index * 0.15, 1.0))
+            return await _run_spec_cv_pipeline_single(
                 image_pil,
                 frame_id=fid,
                 options=options,
             )
-            results.append(item)
-            infer_modes.append(mode)
-            triton_call_count += calls
+
+    outcomes = await asyncio.gather(
+        *[process_one(img, fid, idx) for idx, (img, fid) in enumerate(zip(images, frame_ids))],
+    )
+
+    results: list[dict[str, Any]] = []
+    infer_modes: list[str] = []
+    triton_call_count = 0
+    for item, mode, calls in outcomes:
+        results.append(item)
+        infer_modes.append(mode)
+        triton_call_count += calls
     return CvBatchResult(
         results=results,
         triton_infer_calls=triton_call_count,

workspace/sprint_1_2/CODEBASE/backend/services/triton_runtime_service.py

@@ -25,12 +25,11 @@ INPUT_NAME = "input_image"
 OUTPUT_NAME = "logits"
 
 TRITON_INFER_TIMEOUT = float(os.getenv("TRITON_INFER_TIMEOUT", "120"))
-TRITON_INFER_RETRIES = int(os.getenv("TRITON_INFER_RETRIES", "5"))
-TRITON_RETRY_BASE_S = float(os.getenv("TRITON_RETRY_BASE_S", "4"))
+TRITON_INFER_RETRIES = int(os.getenv("TRITON_INFER_RETRIES", "2"))
+TRITON_RETRY_BASE_S = float(os.getenv("TRITON_RETRY_BASE_S", "2.0"))
 TRITON_USE_BATCH_INFER = os.getenv("TRITON_USE_BATCH_INFER", "true").lower()
 RETRYABLE_STATUS = {429, 502, 503, 504}
 
-_triton_infer_lock = asyncio.Lock()
 _adapter: TritonAdapter | None = None
 _adapter_endpoint: str | None = None
 
@@ -43,6 +42,11 @@ def _get_adapter() -> TritonAdapter:
     global _adapter, _adapter_endpoint
     endpoint = get_triton_endpoint()
     if _adapter is None or _adapter_endpoint != endpoint:
+        if _adapter is not None:
+            try:
+                asyncio.get_event_loop().run_until_complete(_adapter.close())
+            except RuntimeError:
+                pass
         _adapter = TritonAdapter(endpoint_url=endpoint, timeout=TRITON_INFER_TIMEOUT)
         _adapter_endpoint = endpoint
     return _adapter
@@ -201,7 +205,7 @@ def _infer_angle_logits_chunk(images: list[Image.Image], model_name: str) -> tup
     if not _should_try_batched_infer(len(images)):
         return _infer_angle_logits_sequential(images, model_name), "sequential"
     try:
-        return _infer_angle_logits_batch(images, model_name, max_retries=1), "batched"
+        return _infer_angle_logits_batch(images, model_name), "batched"
     except Exception as exc:
         logger.warning(
             "Batched angle infer×%s failed (%s); falling back to sequential single-image calls",
@@ -249,7 +253,7 @@ def _infer_inflammation_logits_chunk(
     if not _should_try_batched_infer(len(images)):
         return _infer_inflammation_logits_sequential(images, model_name), "sequential"
     try:
-        return _infer_inflammation_logits_batch(images, model_name, max_retries=1), "batched"
+        return _infer_inflammation_logits_batch(images, model_name), "batched"
     except Exception as exc:
         logger.warning(
             "Batched inflammation infer×%s failed (%s); falling back to sequential",
@@ -297,7 +301,7 @@ def _infer_segmentation_logits_chunk(
     if not _should_try_batched_infer(len(images)):
         return _infer_segmentation_logits_sequential(images, model_name), "sequential"
     try:
-        return _infer_segmentation_logits_batch(images, model_name, max_retries=1), "batched"
+        return _infer_segmentation_logits_batch(images, model_name), "batched"
     except Exception as exc:
         logger.warning(
             "Batched segmentation infer×%s failed (%s); falling back to sequential",
@@ -311,61 +315,57 @@ async def infer_angle_logits(
     images: list[Image.Image],
     model_name: str,
 ) -> tuple[np.ndarray, Literal["batched", "sequential"], int]:
-    """Run angle classification under the global Triton lock. Returns (logits, mode, call_count)."""
-    async with _triton_infer_lock:
-        all_logits: list[np.ndarray] = []
-        modes: list[str] = []
-        call_count = 0
-        for chunk in chunk_sequence(images):
-            logits, mode = await asyncio.to_thread(_infer_angle_logits_chunk, chunk, model_name)
-            all_logits.append(logits)
-            modes.append(mode)
-            call_count += 1 if mode == "batched" else len(chunk)
-        combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
-        infer_mode: Literal["batched", "sequential"] = (
-            "sequential" if any(m == "sequential" for m in modes) else "batched"
-        )
-        return combined, infer_mode, call_count
+    all_logits: list[np.ndarray] = []
+    modes: list[str] = []
+    call_count = 0
+    for chunk in chunk_sequence(images):
+        logits, mode = await asyncio.to_thread(_infer_angle_logits_chunk, chunk, model_name)
+        all_logits.append(logits)
+        modes.append(mode)
+        call_count += 1 if mode == "batched" else len(chunk)
+    combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
+    infer_mode: Literal["batched", "sequential"] = (
+        "sequential" if any(m == "sequential" for m in modes) else "batched"
+    )
+    return combined, infer_mode, call_count
 
 
 async def infer_inflammation_logits(
     images: list[Image.Image],
     model_name: str,
 ) -> tuple[np.ndarray, Literal["batched", "sequential"], int]:
-    async with _triton_infer_lock:
-        all_logits: list[np.ndarray] = []
-        modes: list[str] = []
-        call_count = 0
-        for chunk in chunk_sequence(images):
-            logits, mode = await asyncio.to_thread(_infer_inflammation_logits_chunk, chunk, model_name)
-            all_logits.append(logits)
-            modes.append(mode)
-            call_count += 1 if mode == "batched" else len(chunk)
-        combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
-        infer_mode: Literal["batched", "sequential"] = (
-            "sequential" if any(m == "sequential" for m in modes) else "batched"
-        )
-        return combined, infer_mode, call_count
+    all_logits: list[np.ndarray] = []
+    modes: list[str] = []
+    call_count = 0
+    for chunk in chunk_sequence(images):
+        logits, mode = await asyncio.to_thread(_infer_inflammation_logits_chunk, chunk, model_name)
+        all_logits.append(logits)
+        modes.append(mode)
+        call_count += 1 if mode == "batched" else len(chunk)
+    combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
+    infer_mode: Literal["batched", "sequential"] = (
+        "sequential" if any(m == "sequential" for m in modes) else "batched"
+    )
+    return combined, infer_mode, call_count
 
 
 async def infer_segmentation_logits(
     images: list[Image.Image],
     model_name: str,
 ) -> tuple[np.ndarray, Literal["batched", "sequential"], int]:
-    async with _triton_infer_lock:
-        all_logits: list[np.ndarray] = []
-        modes: list[str] = []
-        call_count = 0
-        for chunk in chunk_sequence(images):
-            logits, mode = await asyncio.to_thread(_infer_segmentation_logits_chunk, chunk, model_name)
-            all_logits.append(logits)
-            modes.append(mode)
-            call_count += 1 if mode == "batched" else len(chunk)
-        combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
-        infer_mode: Literal["batched", "sequential"] = (
-            "sequential" if any(m == "sequential" for m in modes) else "batched"
-        )
-        return combined, infer_mode, call_count
+    all_logits: list[np.ndarray] = []
+    modes: list[str] = []
+    call_count = 0
+    for chunk in chunk_sequence(images):
+        logits, mode = await asyncio.to_thread(_infer_segmentation_logits_chunk, chunk, model_name)
+        all_logits.append(logits)
+        modes.append(mode)
+        call_count += 1 if mode == "batched" else len(chunk)
+    combined = np.concatenate(all_logits, axis=0) if len(all_logits) > 1 else all_logits[0]
+    infer_mode: Literal["batched", "sequential"] = (
+        "sequential" if any(m == "sequential" for m in modes) else "batched"
+    )
+    return combined, infer_mode, call_count
 
 
 async def infer_angle_logits_single(image: Image.Image, model_name: str) -> tuple[np.ndarray, str, int]:

workspace/sprint_1_2/CODEBASE/backend/services/triton_warmup.py

@@ -1,6 +1,7 @@
 """Warm up Modal Triton on CV service startup — reduces cold-start 502s and first-frame latency."""
 from __future__ import annotations
 
+import asyncio
 import logging
 import os
 
@@ -23,6 +24,20 @@ def _warmup_model_versions() -> dict[str, str]:
     return versions
 
 
+async def _warmup_one(
+    name: str,
+    coro,
+    timeout: float,
+) -> None:
+    try:
+        await asyncio.wait_for(coro, timeout=timeout)
+        logger.info("Triton warmup %s complete", name)
+    except asyncio.TimeoutError:
+        logger.warning("Triton warmup %s timed out after %.1fs", name, timeout)
+    except Exception as exc:
+        logger.warning("Triton warmup %s failed: %s", name, exc)
+
+
 async def warmup_triton_models() -> None:
     if os.getenv("CV_SKIP_TRITON_WARMUP", "").lower() in {"1", "true", "yes"}:
         logger.info("Triton warmup skipped (CV_SKIP_TRITON_WARMUP)")
@@ -36,13 +51,29 @@ async def warmup_triton_models() -> None:
     img224 = Image.new("RGB", (224, 224), color=(128, 128, 128))
     img512 = Image.new("RGB", (512, 512), color=(128, 128, 128))
 
+    warmup_timeout = float(os.getenv("TRITON_WARMUP_TIMEOUT", "15"))
+
     logger.info(
-        "Warming up Triton (angle=%s, inflammation=%s, segmentation=%s)…",
+        "Warming up Triton (angle=%s, inflammation=%s, segmentation=%s, timeout=%.1fs)…",
         angle_model,
         inflam_model,
         seg_model,
+        warmup_timeout,
+    )
+
+    await _warmup_one(
+        "angle",
+        triton_runtime.infer_angle_logits_single(img224, angle_model),
+        warmup_timeout,
+    )
+    await _warmup_one(
+        "inflammation",
+        triton_runtime.infer_inflammation_logits_single(img224, inflam_model),
+        warmup_timeout,
+    )
+    await _warmup_one(
+        "segmentation",
+        triton_runtime.infer_segmentation_logits_single(img512, seg_model),
+        warmup_timeout,
     )
-    await triton_runtime.infer_angle_logits_single(img224, angle_model)
-    await triton_runtime.infer_inflammation_logits_single(img224, inflam_model)
-    await triton_runtime.infer_segmentation_logits_single(img512, seg_model)
     logger.info("Triton warmup complete")

workspace/sprint_1_2/CODEBASE/backend/start_celery_workers.sh

@@ -0,0 +1,179 @@
+#!/usr/bin/env bash
+#
+# Start Redis + Celery worker for CV inference experiments.
+#
+# Usage:
+#   ./backend/start_celery_workers.sh          # start both
+#   ./backend/start_celery_workers.sh stop     # stop both
+#   ./backend/start_celery_workers.sh restart # restart both
+#   ./backend/start_celery_workers.sh status  # show status
+#
+# Logs:
+#   logs/redis.log
+#   logs/celery.log
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CODEBASE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+LOG_DIR="${CODEBASE_ROOT}/logs"
+REDIS_PID_FILE="${LOG_DIR}/redis.pid"
+CELERY_PID_FILE="${LOG_DIR}/celery.pid"
+
+mkdir -p "${LOG_DIR}"
+
+cd "${CODEBASE_ROOT}"
+export PYTHONPATH="${CODEBASE_ROOT}:${PYTHONPATH:-}"
+
+REDIS_PORT="${REDIS_PORT:-6379}"
+REDIS_DB="${REDIS_DB:-0}"
+CELERY_CHUNK_SIZE="${CELERY_CHUNK_SIZE:-4}"
+export TRITON_ENDPOINT="${TRITON_ENDPOINT:-https://dtj-tran--triton-s3-service-unified-triton-server.modal.run}"
+
+
+is_redis_running() {
+    if [ -f "${REDIS_PID_FILE}" ]; then
+        local pid
+        pid=$(cat "${REDIS_PID_FILE}")
+        if kill -0 "${pid}" 2>/dev/null; then
+            return 0
+        else
+            rm -f "${REDIS_PID_FILE}"
+        fi
+    fi
+    return 1
+}
+
+
+start_redis() {
+    if is_redis_running; then
+        echo "Redis already running (PID $(cat ${REDIS_PID_FILE}))"
+        return 0
+    fi
+
+    echo "Starting Redis on port ${REDIS_PORT}..."
+    redis-server \
+        --port "${REDIS_PORT}" \
+        --daemonize yes \
+        --pidfile "${REDIS_PID_FILE}" \
+        --logfile "${LOG_DIR}/redis.log" \
+        --save "" \
+        --appendonly no
+
+    sleep 0.5
+
+    if is_redis_running; then
+        echo "Redis started (PID $(cat ${REDIS_PID_FILE}))"
+    else
+        echo "Redis failed to start. Check ${LOG_DIR}/redis.log"
+        exit 1
+    fi
+}
+
+
+start_celery() {
+    if [ -f "${CELERY_PID_FILE}" ] && kill -0 "$(cat "${CELERY_PID_FILE}")" 2>/dev/null; then
+        echo "Celery already running (PID $(cat ${CELERY_PID_FILE}))"
+        return 0
+    fi
+
+    echo "Starting Celery worker..."
+    nohup celery \
+        -A backend.services.celery_app \
+        worker \
+        --loglevel=info \
+        -Q cv-inference \
+        --concurrency=2 \
+    > "${LOG_DIR}/celery.log" 2>&1 &
+
+    echo $! > "${CELERY_PID_FILE}"
+
+    sleep 1
+
+    if kill -0 "$(cat "${CELERY_PID_FILE}")" 2>/dev/null; then
+        echo "Celery started (PID $(cat ${CELERY_PID_FILE}))"
+        echo "  Logs: ${LOG_DIR}/celery.log"
+    else
+        echo "Celery failed to start. Check ${LOG_DIR}/celery.log"
+        rm -f "${CELERY_PID_FILE}"
+        exit 1
+    fi
+}
+
+
+stop_redis() {
+    if is_redis_running; then
+        local pid
+        pid=$(cat "${REDIS_PID_FILE}")
+        echo "Stopping Redis (PID ${pid})..."
+        kill "${pid}" 2>/dev/null || true
+        sleep 0.5
+        rm -f "${REDIS_PID_FILE}"
+        echo "Redis stopped"
+    else
+        echo "Redis not running"
+    fi
+}
+
+
+stop_celery() {
+    if [ -f "${CELERY_PID_FILE}" ] && kill -0 "$(cat "${CELERY_PID_FILE}")" 2>/dev/null; then
+        local pid
+        pid=$(cat "${CELERY_PID_FILE}")
+        echo "Stopping Celery (PID ${pid})..."
+        kill "${pid}" 2>/dev/null || true
+        sleep 1
+        # Force kill if still running
+        if kill -0 "${pid}" 2>/dev/null; then
+            kill -9 "${pid}" 2>/dev/null || true
+        fi
+        rm -f "${CELERY_PID_FILE}"
+        echo "Celery stopped"
+    else
+        echo "Celery not running"
+    fi
+}
+
+
+status() {
+    echo "=== Worker Status ==="
+    if is_redis_running; then
+        echo "Redis:   running (PID $(cat ${REDIS_PID_FILE}))"
+    else
+        echo "Redis:   stopped"
+    fi
+
+    if [ -f "${CELERY_PID_FILE}" ] && kill -0 "$(cat "${CELERY_PID_FILE}")" 2>/dev/null; then
+        echo "Celery:  running (PID $(cat ${CELERY_PID_FILE}))"
+    else
+        echo "Celery:  stopped"
+    fi
+}
+
+
+case "${1:-start}" in
+    start)
+        start_redis
+        start_celery
+        echo ""
+        echo "Workers ready. Test with:"
+        echo "  curl http://localhost:8001/api/test/analyze/batch/celery"
+        ;;
+    stop)
+        stop_celery
+        stop_redis
+        ;;
+    restart)
+        stop_celery
+        stop_redis
+        start_redis
+        start_celery
+        ;;
+    status)
+        status
+        ;;
+    *)
+        echo "Usage: $0 {start|stop|restart|status}"
+        exit 1
+        ;;
+esac