msk_admin/Lumina-MSK
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3 from DTJ-Tran/poc1

Browse Source 
Poc1-Proof of Concept verison 1
This commit is contained in:
David Tran
2026-07-07 15:56:36 +07:00
committed by GitHub
parent 41e6a2a416 1622dc8fc5
commit 2484597c7a
453 changed files with 84097 additions and 66425 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
workspace/sprint_1_2/CODEBASE/infra/cloud_llm_gateway/Dockerfile Normal file
View File

@@ -0,0 +1,11 @@
FROM python:3.12-slim
RUN pip install fastapi[standard] uvicorn httpx transformers torch
WORKDIR /app
COPY modal_endpoint.py /app/modal_endpoint.py
EXPOSE 8000
CMD ["uvicorn", "modal_endpoint:app", "--host", "0.0.0.0", "--port", "8000"]

8
workspace/sprint_1_2/CODEBASE/infra/cloud_llm_gateway/modal_config.toml Normal file
View File

@@ -0,0 +1,8 @@
app_name = "vkist-medgemma"
function = "medgemma_endpoint"
gpu = "T4"
timeout = 30
min_containers = 1
max_containers = 5
scaledown_window = 300
allow_concurrent_inputs = 10

126
workspace/sprint_1_2/CODEBASE/infra/cloud_llm_gateway/modal_endpoint.py Normal file
View File

@@ -0,0 +1,126 @@
import modal
import os
from pathlib import Path
import logging
logger = logging.getLogger(__name__)
SECRETS_DIR = Path(__file__).resolve().parent.parent.parent.parent / "secrets"
def _load_secret(name: str, filename: str) -> str:
file_path = SECRETS_DIR / filename
env_file = os.getenv(f"{name}_FILE")
if env_file:
resolved = Path(env_file)
if resolved.exists():
with open(resolved, "r", encoding="utf-8") as f:
return f.read().strip()
if file_path.exists():
with open(file_path, "r", encoding="utf-8") as f:
return f.read().strip()
raise RuntimeError(
f"Required secret {name} not found at {file_path} or via {name}_FILE env var"
)
image = (
modal.Image.debian_slim()
.pip_install(
"fastapi[standard]",
"uvicorn",
"httpx",
"transformers",
"torch",
"sentencepiece",
"protobuf",
)
.env({"MODAL_API_KEY": os.getenv("MODAL_API_KEY", "")})
)
app = modal.App("vkist-medgemma", image=image)
MEDGEMMA_MODEL = os.getenv("MEDGEMMA_MODEL", "medgemma-large-24b")
PORT = int(os.getenv("PORT", "8000"))
MEDGEMMA_API_KEY = _load_secret("MEDGEMMA_API_KEY", "modal_api_key.txt")
@app.function(
gpu="T4",
timeout=30,
min_containers=1,
max_containers=5,
scaledown_window=300,
allow_concurrent_inputs=10,
secrets=[
modal.Secret.from_name("gcp-secrets"),
modal.Secret.from_name("modal-api-key"),
],
)
@modal.asgi_app()
def medgemma_endpoint():
from fastapi import FastAPI, Request, HTTPException, status
from fastapi.responses import JSONResponse, StreamingResponse
import httpx
import json
web_app = FastAPI(title="MedGemma Cloud Endpoint")
async def _verify_api_key(request: Request):
key = request.headers.get("X-Modal-Key", "")
if not MEDGEMMA_API_KEY or key != MEDGEMMA_API_KEY:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Invalid or missing API key",
)
@web_app.get("/health")
async def health():
return {"status": "ok", "model": MEDGEMMA_MODEL}
@web_app.post("/api/generate")
async def generate(request: Request):
await _verify_api_key(request)
body = await request.json()
prompt = body.get("prompt", "")
stream = body.get("stream", False)
model = body.get("model", MEDGEMMA_MODEL)
if stream:
return StreamingResponse(
_stream_generate(prompt, model),
media_type="text/event-stream",
)
async with httpx.AsyncClient(timeout=30.0) as client:
response = await client.post(
"http://localhost:11434/api/generate",
json={
"model": model,
"prompt": prompt,
"stream": False,
"options": body.get("options", {}),
},
)
response.raise_for_status()
return JSONResponse(response.json())
async def _stream_generate(prompt: str, model: str):
async with httpx.AsyncClient(timeout=30.0) as client:
async with client.stream(
"POST",
"http://localhost:11434/api/generate",
json={
"model": model,
"prompt": prompt,
"stream": True,
"options": {"temperature": 0.1, "top_p": 0.8, "top_k": 40, "num_predict": 2048},
},
) as response:
async for line in response.aiter_lines():
if line.startswith("data:"):
yield line
return web_app
if __name__ == "__main__":
app.serve()

132
workspace/sprint_1_2/CODEBASE/infra/implementation/ollama/medGemma_modal_ollama_deploy.py Normal file
View File

@@ -0,0 +1,132 @@
import subprocess
import time
import requests
import modal
OLLAMA_PORT = 11434
MODEL_NAME = "medgemma:4b"
# Persist downloaded models
ollama_volume = modal.Volume.from_name(
"ollama-model-cache",
create_if_missing=True,
)
image = (
modal.Image.debian_slim()
.pip_install("requests") # <--- THIS IS MISSING
.run_commands(
"apt-get update",
"apt-get install -y curl zstd ca-certificates",
"curl -fsSL https://ollama.com/install.sh | sh",
)
.env(
{
"OLLAMA_HOST": "0.0.0.0"
}
)
)
app = modal.App("ollama-medgemma")
@app.cls(
image=image,
gpu="l4",
min_containers=1,
max_containers=3,
scaledown_window=300,
volumes={"/root/.ollama": ollama_volume},
)
class OllamaServer:
@modal.enter()
def start_server(self):
print("Starting Ollama server...")
self.process = subprocess.Popen(
["ollama", "serve"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
try:
self.wait_until_ready()
print(f"Checking model: {MODEL_NAME}")
installed = subprocess.run(
["ollama", "list"],
capture_output=True,
text=True,
check=True,
)
if MODEL_NAME not in installed.stdout:
print(f"Downloading {MODEL_NAME}...")
subprocess.run(
["ollama", "pull", MODEL_NAME],
check=True,
)
else:
print("Model already cached.")
print("Ollama ready.")
response = requests.post(
"http://0.0.0.0:11434/api/generate",
json={
"model": MODEL_NAME,
"prompt": "",
"stream": False,
"keep_alive": -1,
},
timeout=120,
)
response.raise_for_status()
print(response.json())
except Exception:
self.process.kill()
raise
def wait_until_ready(self):
deadline = time.time() + 180
while time.time() < deadline:
try:
r = requests.get(
f"http://0.0.0.0:{OLLAMA_PORT}/api/tags",
timeout=2,
)
if r.status_code == 200:
return
except requests.RequestException:
pass
time.sleep(1)
raise RuntimeError("Ollama failed to start.")
@modal.web_server(port=OLLAMA_PORT)
def web(self):
pass
@modal.exit()
def shutdown(self):
print("Stopping Ollama...")
if self.process.poll() is None:
self.process.terminate()
try:
self.process.wait(timeout=10)
except subprocess.TimeoutExpired:
self.process.kill()

6
workspace/sprint_1_2/CODEBASE/infra/implementation/triton_run/modal_triton.py
View File

@@ -188,6 +188,12 @@ async def proxy_all_triton_request(path: str, request: Request):
status_code=502
)
@web_app.get("/v2/models")
async def forward_list_models():
async with httpx.AsyncClient() as client:
r = await client.get("http://127.0.0.1:8000/v2/models")
return Response(content=r.content, status_code=r.status_code, media_type=r.headers.get("content-type"))
# -------------------------------------------------------------
# THE UNIFIED SERVICE FUNCTION (1 Container, 1 GPU, 1 Triton Process)
# -------------------------------------------------------------

2
workspace/sprint_1_2/CODEBASE/infra/implementation/triton_run/run.sh
View File

@@ -1,2 +1,2 @@
cd ../PILOT_PROJECT/workspace/sprint_1_2/CODEBASE/infra/implementation/triton_run
modal deploy PILOT_PROJECT/workspace/sprint_1_2/codebase/infra/implementation/triton_run/modal_triton.py
MODAL_PROFILE=dtj-tran modal deploy PILOT_PROJECT/workspace/sprint_1_2/codebase/infra/implementation/triton_run/modal_triton.py

4
workspace/sprint_1_2/CODEBASE/infra/spec/infra_spec.md
View File

@@ -18,9 +18,11 @@ Platform Engineering Team
- NGINX configured as ingress controller with SSL/TLS termination, path-based routing to backend services, and WebSocket support for real-time features.
- Keepalived deployed in active-passive mode across cluster nodes, assigning a virtual IP (VIP) for seamless failover.
- PostgreSQL and Redis instances are provisioned and managed via Terraform; connection details are exposed as environment variables to consuming rooms.
- Cloud LLM Gateway (FastAPI) provisioned in K3s for NFR-16a governed egress: routes to GCP Vertex AI (Gemini) and Modal (MedGemma) with mandatory redaction, consent, and audit logging.
- Foundational logging: structured JSON logs shipped to centralized observability stack (outside scope of this spec).
- Security: network policies restrict inter-room communication to declared interfaces; NGINX enforces authentication headers and rate limits.
- Security: network policies restrict inter-room communication to declared interfaces; NGINX enforces authentication headers and rate limits. All cloud egress flows through Cloud LLM Gateway — no direct client-to-cloud calls permitted.
- Observability: exposes Prometheus metrics endpoints; health checks for liveness and readiness.
- NFR-16a Cost Guard: Cloud LLM Gateway tracks MedGemma usage count against total consult sessions; alert triggers if MedGemma ratio exceeds 20% of sessions over a rolling 24h window.
## Interface Contract
See `infra/spec/interface-contract.md`.

11
workspace/sprint_1_2/CODEBASE/infra/tests/test_1_model.py
View File

@@ -28,6 +28,17 @@ def preprocess_224(img: Image.Image) -> np.ndarray:
arr = np.expand_dims(arr, axis=0) # Add batch dim -> NCHW
return arr
def preprocess_512(img: Image.Image) -> np.ndarray:
"""Preprocesses image to NCHW FP32 [1, 3, 512, 512] matching ResNet50 input requirements"""
img_resized = img.resize((512, 512), Image.Resampling.BILINEAR)
arr = np.asarray(img_resized).astype(np.float32) / 255.0
mean = np.array([0.485, 0.456, 0.406], dtype=np.float32)
std = np.array([0.229, 0.224, 0.225], dtype=np.float32)
arr = (arr - mean) / std
arr = arr.transpose(2, 0, 1) # HWC -> CHW
arr = np.expand_dims(arr, axis=0) # Add batch dim -> NCHW
return arr
def softmax(x: np.ndarray) -> np.ndarray:
e = np.exp(x - np.max(x))
return e / np.sum(e)