test_workflow
Some checks failed
Some checks failed
This commit is contained in:
@@ -0,0 +1,118 @@
|
||||
# =============================================================================
|
||||
# Stage 1: Wheelhouse Builder
|
||||
# Pre-compiles all Python dependencies into .whl files into /wheels.
|
||||
# This layer is cached independently and only rebuilds when requirements.txt
|
||||
# changes. Heavy packages (torch, opencv, pycocotools, etc.) are compiled once
|
||||
# per platform (amd64/arm64) and reused by all subsequent builds.
|
||||
# =============================================================================
|
||||
FROM python:3.12-slim-bookworm AS wheelhouse
|
||||
|
||||
WORKDIR /wheels
|
||||
|
||||
# Install build tools required for packages with C extensions
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libxcb1 \
|
||||
libx11-6 \
|
||||
libxext6 \
|
||||
libsm6 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy requirements first so this layer is cached unless deps change
|
||||
COPY requirements.txt .
|
||||
|
||||
# Build wheel files for all dependencies AND their transitive dependencies.
|
||||
# --wheel-dir=/wheels: write .whl files to the wheelhouse directory
|
||||
# (no --no-deps: we want the full dependency tree pre-compiled)
|
||||
RUN pip wheel --no-cache-dir --wheel-dir=/wheels -r requirements.txt
|
||||
|
||||
# =============================================================================
|
||||
# Stage 2: Application Builder
|
||||
# Creates a venv and installs dependencies from the local wheelhouse.
|
||||
# Because pip installs from /wheels (not PyPI), this layer stays cached as
|
||||
# long as the wheelhouse is unchanged — application code changes do NOT
|
||||
# trigger re-downloading or recompiling any dependency.
|
||||
# =============================================================================
|
||||
FROM python:3.12-slim-bookworm AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install build tools (kept for safety; wheels are pre-compiled but some
|
||||
# packages may still invoke build steps during install)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
g++ \
|
||||
make \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libxcb1 \
|
||||
libx11-6 \
|
||||
libxext6 \
|
||||
libsm6 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Create virtual environment so we can copy a single tree to runtime
|
||||
RUN python -m venv /opt/venv
|
||||
ENV PATH=/opt/venv/bin:$PATH
|
||||
|
||||
# Copy requirements for pip install validation
|
||||
COPY requirements.txt .
|
||||
|
||||
# Copy pre-built wheels from wheelhouse stage (platform-specific)
|
||||
COPY --from=wheelhouse /wheels /wheels
|
||||
|
||||
# Install dependencies FROM LOCAL WHEELS ONLY (no network access needed)
|
||||
# --no-index: do not query PyPI or any external index
|
||||
# --find-links: use only the local wheelhouse at /wheels
|
||||
RUN pip install --no-cache-dir --no-index --find-links=/wheels -r requirements.txt
|
||||
|
||||
# =============================================================================
|
||||
# Stage 3: Runtime
|
||||
# Minimal image containing only the venv and application code.
|
||||
# =============================================================================
|
||||
FROM python:3.12-slim-bookworm AS runtime
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Create non-root user
|
||||
# RUN groupadd -r appgroup && useradd -r -g appgroup -d /app -s /sbin/nologin appuser
|
||||
|
||||
# # Copy the entire venv from builder
|
||||
# COPY --from=builder --chown=appuser:appgroup /opt/venv /opt/venv
|
||||
# Step 2: Assign an explicit numeric ID (like 999) when creating the user/group
|
||||
RUN groupadd -g 999 appgroup && \
|
||||
useradd -r -u 999 -g appgroup -d /app -s /sbin/nologin appuser
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libxcb1 \
|
||||
libx11-6 \
|
||||
libxext6 \
|
||||
libsm6 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
# Step 3: Use the exact matching numeric IDs for --chown
|
||||
COPY --from=builder --chown=999:999 /opt/venv /opt/venv
|
||||
ENV PATH=/opt/venv/bin:$PATH
|
||||
|
||||
# Copy application code
|
||||
COPY --chown=999:999 backend ./backend
|
||||
|
||||
# Switch to non-root user
|
||||
USER appuser
|
||||
|
||||
# Expose port
|
||||
EXPOSE 8001
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')" || exit 1
|
||||
|
||||
# Run the CV inference server
|
||||
ENTRYPOINT ["python", "-m", "backend.cv_inference_server"]
|
||||
@@ -0,0 +1,22 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
cv-inference-server:
|
||||
image: public.ecr.aws/${ECR_PUBLIC_REGISTRY_ALIAS}/msk-cv-inference-server:latest
|
||||
container_name: cv-inference-server
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8001:8001"
|
||||
# Environment variables passed from shell (injected by webhook listener from Gitea vars)
|
||||
# No .env file needed — all config comes from Gitea repo variables via webhook payload
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: pilot-network
|
||||
external: true
|
||||
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
Modal script to build and push VKIST backend Docker image to container registry.
|
||||
|
||||
Run from Gitea runner on Lightsail (2GB RAM) - offloads heavy build to Modal serverless.
|
||||
|
||||
Usage:
|
||||
modal run gitea_modal_build.py \
|
||||
--registry public.ecr.aws/vkist-project \
|
||||
--image-name vkist-backend \
|
||||
--tag v1.2.3 \
|
||||
--push
|
||||
|
||||
# Or deploy as reusable Modal app:
|
||||
modal deploy gitea_modal_build.py
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import modal
|
||||
|
||||
# =============================================================================
|
||||
# Configuration - Dynamic paths relative to this script
|
||||
# =============================================================================
|
||||
|
||||
# Get the directory where this script actually lives on the machine running it
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
|
||||
# Automatically resolve the root of the repository (climbing up 3 levels)
|
||||
# From: deps/implementation/backend_deploy/gitea_modal_build.py
|
||||
# To: CODEBASE/
|
||||
PROJECT_ROOT = SCRIPT_DIR.parents[2]
|
||||
|
||||
# Explicitly find your local Dockerfile
|
||||
DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile"
|
||||
|
||||
print(f"--- Environment Verification ---")
|
||||
print(f"Script location: {SCRIPT_DIR}")
|
||||
print(f"Resolved PROJECT_ROOT: {PROJECT_ROOT} (Exists: {PROJECT_ROOT.exists()})")
|
||||
print(f"Resolved DOCKERFILE_PATH: {DOCKERFILE_PATH} (Exists: {DOCKERFILE_PATH.exists()})")
|
||||
print(f"----------------------------------------")
|
||||
|
||||
# Project layout variables
|
||||
BACKEND_SOURCE = "backend"
|
||||
REQUIREMENTS_FILE = "requirements.txt"
|
||||
|
||||
# Default registry/image settings
|
||||
DEFAULT_REGISTRY = "public.ecr.aws/i9a4e3f6/msk-cv-inference-server"
|
||||
DEFAULT_IMAGE_NAME = "msk-lumina-backend"
|
||||
DEFAULT_TAG = "latest"
|
||||
|
||||
# Modal app configuration
|
||||
APP_NAME = "msk-lumina-backend-builder"
|
||||
|
||||
# =============================================================================
|
||||
# Modal Image Definition - uses the project's Dockerfile for consistency
|
||||
# =============================================================================
|
||||
|
||||
# Build the image using the existing multi-stage Dockerfile
|
||||
# We set context_dir to PROJECT_ROOT so the Dockerfile can see ../../../../backend
|
||||
backend_image = modal.Image.from_dockerfile(
|
||||
str(DOCKERFILE_PATH),
|
||||
context_dir=str(PROJECT_ROOT),
|
||||
).pip_install(
|
||||
"docker", # for test_image function
|
||||
"boto3", # for ECR auth if needed
|
||||
)
|
||||
|
||||
app = modal.App(APP_NAME, image=backend_image)
|
||||
|
||||
# =============================================================================
|
||||
# Build and Push Function
|
||||
# =============================================================================
|
||||
|
||||
@app.function(
|
||||
timeout=900, # 15 min for heavy ML deps (torch, transformers, etc.)
|
||||
cpu=4,
|
||||
memory=8192, # 8GB RAM - plenty for wheel building
|
||||
secrets=[
|
||||
modal.Secret.from_name("aws-secrets"), # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
||||
# modal.Secret.from_name("dockerhub-credentials"), # DOCKERHUB_USERNAME, DOCKERHUB_TOKEN (optional)
|
||||
# modal.Secret.from_name("gcp-artifact-registry"), # GCP credentials (optional)
|
||||
],
|
||||
)
|
||||
def build_and_push(
|
||||
registry: str = DEFAULT_REGISTRY,
|
||||
image_name: str = DEFAULT_IMAGE_NAME,
|
||||
tag: str = DEFAULT_TAG,
|
||||
push: bool = True,
|
||||
platform: str = "linux/amd64",
|
||||
) -> dict:
|
||||
"""
|
||||
Build the backend Docker image and push to registry.
|
||||
|
||||
Args:
|
||||
registry: Container registry hostname (e.g., public.ecr.aws/vkist-project)
|
||||
image_name: Image name (e.g., vkist-backend)
|
||||
tag: Image tag (e.g., v1.2.3, latest, git-sha)
|
||||
push: Whether to push to registry
|
||||
platform: Target platform (linux/amd64, linux/arm64)
|
||||
|
||||
Returns:
|
||||
Dict with image reference, digest, and size info
|
||||
"""
|
||||
import docker
|
||||
|
||||
full_image = f"{registry}/{image_name}:{tag}"
|
||||
print(f"Building {full_image} for {platform}")
|
||||
|
||||
# Initialize Docker client (uses Modal's container runtime)
|
||||
client = docker.from_env()
|
||||
|
||||
# Build using the Dockerfile from project root context
|
||||
print(f"Build context: {PROJECT_ROOT}")
|
||||
print(f"Dockerfile: {DOCKERFILE_PATH}")
|
||||
|
||||
# Build the image
|
||||
image, build_logs = client.images.build(
|
||||
path=PROJECT_ROOT,
|
||||
dockerfile=DOCKERFILE_PATH,
|
||||
tag=full_image,
|
||||
platform=platform,
|
||||
rm=True,
|
||||
pull=True,
|
||||
buildargs={
|
||||
"BUILDKIT_INLINE_CACHE": "1",
|
||||
},
|
||||
)
|
||||
|
||||
# Stream build logs
|
||||
for chunk in build_logs:
|
||||
if "stream" in chunk:
|
||||
print(chunk["stream"].strip())
|
||||
elif "error" in chunk:
|
||||
raise RuntimeError(f"Build failed: {chunk['error']}")
|
||||
|
||||
print(f"Build complete: {image.id[:12]}")
|
||||
|
||||
# Get image details
|
||||
image_info = client.images.get(full_image)
|
||||
size_bytes = image_info.attrs["Size"]
|
||||
digest = image_info.attrs.get("RepoDigests", ["unknown"])[0]
|
||||
|
||||
result = {
|
||||
"image": full_image,
|
||||
"digest": digest,
|
||||
"size_bytes": size_bytes,
|
||||
"image_id": image.id,
|
||||
}
|
||||
|
||||
if push:
|
||||
print(f"Pushing to {registry}...")
|
||||
|
||||
# Authenticate with registry based on hostname
|
||||
_authenticate_registry(registry, client)
|
||||
|
||||
# Push image
|
||||
push_logs = client.images.push(
|
||||
repository=f"{registry}/{image_name}",
|
||||
tag=tag,
|
||||
stream=True,
|
||||
decode=True,
|
||||
)
|
||||
|
||||
for chunk in push_logs:
|
||||
if "status" in chunk:
|
||||
print(f" {chunk['status']}: {chunk.get('progress', '')}")
|
||||
if "error" in chunk:
|
||||
raise RuntimeError(f"Push failed: {chunk['error']}")
|
||||
|
||||
print(f"Push complete: {full_image}")
|
||||
|
||||
# Get digest after push
|
||||
pushed_image = client.images.get(full_image)
|
||||
result["digest"] = pushed_image.attrs.get("RepoDigests", ["unknown"])[0]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _authenticate_registry(registry: str, client):
|
||||
"""Authenticate Docker client with the target registry."""
|
||||
import subprocess
|
||||
|
||||
if "ecr.aws" in registry or "ecr." in registry:
|
||||
# AWS ECR (public or private)
|
||||
region = os.getenv("AWS_REGION", "us-east-1")
|
||||
print(f"Authenticating with AWS ECR ({region})...")
|
||||
subprocess.run([
|
||||
"aws", "ecr", "get-login-password", "--region", region
|
||||
], check=True, capture_output=True)
|
||||
# Note: Modal's aws-ecr-credentials secret should handle this via env vars
|
||||
|
||||
elif "docker.io" in registry or "index.docker.io" in registry or "/" not in registry.split("/")[0]:
|
||||
# Docker Hub
|
||||
username = os.getenv("DOCKERHUB_USERNAME")
|
||||
password = os.getenv("DOCKERHUB_TOKEN")
|
||||
if username and password:
|
||||
print("Authenticating with Docker Hub...")
|
||||
client.login(username=username, password=password)
|
||||
|
||||
elif "gcr.io" in registry or "pkg.dev" in registry:
|
||||
# Google Container Registry / Artifact Registry
|
||||
print("Authenticating with GCP Artifact Registry...")
|
||||
subprocess.run(["gcloud", "auth", "configure-docker", "--quiet"], check=True)
|
||||
|
||||
|
||||
@app.function(
|
||||
image=backend_image,
|
||||
timeout=120,
|
||||
secrets=[modal.Secret.from_name("aws-secrets")],
|
||||
)
|
||||
def test_image(registry: str, image_name: str, tag: str) -> dict:
|
||||
"""Test the built image by running a quick health check."""
|
||||
import docker
|
||||
|
||||
full_image = f"{registry}/{image_name}:{tag}"
|
||||
client = docker.from_env()
|
||||
|
||||
print(f"Pulling {full_image} for testing...")
|
||||
client.images.pull(full_image)
|
||||
|
||||
# print("Running health check...")
|
||||
# output = client.containers.run(
|
||||
# full_image,
|
||||
# command=["python", "-c", "import backend.cv_inference_server; print('Import OK')"],
|
||||
# remove=True,
|
||||
# detach=False,
|
||||
# )
|
||||
|
||||
print("Running basic smoke test...")
|
||||
output = client.containers.run(
|
||||
full_image,
|
||||
command=[
|
||||
"python", "-c",
|
||||
"import sys; print('Python', sys.version); "
|
||||
"import cv2; print('OpenCV', cv2.__version__); "
|
||||
# "import backend.cv_inference_server; print('Import OK')",
|
||||
],
|
||||
remove=True,
|
||||
detach=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"output": output.decode().strip(),
|
||||
"image": full_image,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# CLI Entry Point
|
||||
# =============================================================================
|
||||
|
||||
@app.local_entrypoint()
|
||||
def main(
|
||||
registry: str = DEFAULT_REGISTRY,
|
||||
image_name: str = DEFAULT_IMAGE_NAME,
|
||||
tag: str = DEFAULT_TAG,
|
||||
push: bool = True,
|
||||
platform: str = "linux/amd64",
|
||||
test: bool = False,
|
||||
):
|
||||
"""
|
||||
Build and push backend Docker image via Modal.
|
||||
|
||||
Args:
|
||||
registry: Container registry (default: public.ecr.aws/vkist-project)
|
||||
image_name: Image name (default: vkist-backend)
|
||||
tag: Image tag (default: latest)
|
||||
push: Push to registry (default: true)
|
||||
platform: Target platform (default: linux/amd64)
|
||||
test: Run health check after build (default: false)
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("VKIST Backend Docker Build on Modal")
|
||||
print("=" * 60)
|
||||
print(f"Registry: {registry}")
|
||||
print(f"Image: {image_name}")
|
||||
print(f"Tag: {tag}")
|
||||
print(f"Push: {push}")
|
||||
print(f"Platform: {platform}")
|
||||
print(f"Test: {test}")
|
||||
print("=" * 60)
|
||||
|
||||
# Build and push
|
||||
result = build_and_push.remote(
|
||||
registry=registry,
|
||||
image_name=image_name,
|
||||
tag=tag,
|
||||
push=push,
|
||||
platform=platform,
|
||||
)
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("BUILD COMPLETE")
|
||||
print("=" * 60)
|
||||
print(f"Image: {result['image']}")
|
||||
print(f"Digest: {result['digest']}")
|
||||
print(f"Size: {result['size_bytes'] / 1024 / 1024:.1f} MB")
|
||||
print("=" * 60)
|
||||
|
||||
# Output for CI/CD parsing (GitHub Actions / Gitea Actions format)
|
||||
print(f"::set-output name=image::{result['image']}")
|
||||
print(f"::set-output name=digest::{result['digest']}")
|
||||
print(f"::set-output name=size_mb::{result['size_bytes'] / 1024 / 1024:.1f}")
|
||||
|
||||
# Optional test
|
||||
if test and push:
|
||||
print("\nRunning health check...")
|
||||
test_result = test_image.remote(registry, image_name, tag)
|
||||
print(f"Health check: {test_result['status']}")
|
||||
print(f"Output: {test_result['output']}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Allow direct execution for debugging
|
||||
import sys
|
||||
if len(sys.argv) > 1 and sys.argv[1] in ("-h", "--help"):
|
||||
print(__doc__)
|
||||
print("\nExamples:")
|
||||
print(" modal run gitea_modal_build.py --tag v1.0.0")
|
||||
print(" modal run gitea_modal_build.py --registry docker.io/myuser --image-name myapp --tag latest")
|
||||
print(" modal deploy gitea_modal_build.py")
|
||||
else:
|
||||
print("Use: modal run gitea_modal_build.py [options]")
|
||||
print("Or: modal deploy gitea_modal_build.py")
|
||||
110
workspace/sprint_1_2/CODEBASE/deps/implementation/backend_deploy/modal_build.sh
Executable file
110
workspace/sprint_1_2/CODEBASE/deps/implementation/backend_deploy/modal_build.sh
Executable file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Wrapper script for Gitea runner to invoke Modal build.
|
||||
# Runs on Lightsail VM (2GB RAM) - offloads Docker build to Modal serverless.
|
||||
#
|
||||
# Usage: ./modal_build.sh [TAG] [REGISTRY] [IMAGE_NAME] [PLATFORM]
|
||||
# TAG - Image tag (default: git SHA or 'latest')
|
||||
# REGISTRY - Container registry (default: public.ecr.aws/vkist-project)
|
||||
# IMAGE_NAME - Image name (default: vkist-backend)
|
||||
# PLATFORM - Target platform (default: linux/amd64)
|
||||
#
|
||||
# Environment variables (set as Gitea repository secrets):
|
||||
# MODAL_TOKEN - Modal API token (from modal.com/settings)
|
||||
# AWS_ACCESS_KEY_ID - For ECR push
|
||||
# AWS_SECRET_ACCESS_KEY
|
||||
# AWS_REGION - Default: us-east-1
|
||||
# DOCKERHUB_USERNAME - Optional, for Docker Hub
|
||||
# DOCKERHUB_TOKEN - Optional, for Docker Hub
|
||||
#
|
||||
# Example:
|
||||
# ./modal_build.sh v1.2.3
|
||||
# ./modal_build.sh latest public.ecr.aws/my-project my-app linux/arm64
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
MODAL_SCRIPT="${SCRIPT_DIR}/gitea_modal_build.py"
|
||||
|
||||
# Defaults
|
||||
DEFAULT_REGISTRY="public.ecr.aws/vkist-project"
|
||||
DEFAULT_IMAGE_NAME="vkist-backend"
|
||||
DEFAULT_PLATFORM="linux/amd64"
|
||||
|
||||
# Parse arguments
|
||||
TAG="${1:-${GITEA_SHA:-latest}}"
|
||||
REGISTRY="${2:-${DEFAULT_REGISTRY}}"
|
||||
IMAGE_NAME="${3:-${DEFAULT_IMAGE_NAME}}"
|
||||
PLATFORM="${4:-${DEFAULT_PLATFORM}}"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $*"; }
|
||||
|
||||
# Check prerequisites
|
||||
check_prereqs() {
|
||||
log_info "Checking prerequisites..."
|
||||
|
||||
if ! command -v modal &> /dev/null; then
|
||||
log_error "Modal CLI not found. Install with: pip install modal"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$MODAL_SCRIPT" ]; then
|
||||
log_error "Modal script not found at: $MODAL_SCRIPT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check Modal auth
|
||||
if [ -z "${MODAL_TOKEN:-}" ]; then
|
||||
log_warn "MODAL_TOKEN not set. Ensure 'modal setup' was run or token is in environment."
|
||||
fi
|
||||
|
||||
log_info "Prerequisites OK"
|
||||
}
|
||||
|
||||
# Build via Modal
|
||||
build_via_modal() {
|
||||
log_info "Starting Modal build..."
|
||||
log_info "Registry: $REGISTRY"
|
||||
log_info "Image: $IMAGE_NAME"
|
||||
log_info "Tag: $TAG"
|
||||
log_info "Platform: $PLATFORM"
|
||||
|
||||
# Run modal build
|
||||
modal run "$MODAL_SCRIPT" \
|
||||
--registry "$REGISTRY" \
|
||||
--image-name "$IMAGE_NAME" \
|
||||
--tag "$TAG" \
|
||||
--platform "$PLATFORM" \
|
||||
--push
|
||||
|
||||
local exit_code=$?
|
||||
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
log_info "Build and push completed successfully!"
|
||||
log_info "Image: $REGISTRY/$IMAGE_NAME:$TAG"
|
||||
else
|
||||
log_error "Modal build failed with exit code $exit_code"
|
||||
exit $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# Main
|
||||
main() {
|
||||
log_info "=== VKIST Modal Docker Build Wrapper ==="
|
||||
|
||||
check_prereqs
|
||||
build_via_modal
|
||||
|
||||
log_info "=== Done ==="
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# Deploy script for VKIST CV Inference Server (Backend)
|
||||
# Triggered by Gitea webhook — env vars come from webhook payload, NOT .env file
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
PROJECT_DIR="/opt/pilot-project"
|
||||
COMPOSE_FILE="${PROJECT_DIR}/docker-compose.backend.yaml"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Starting CV inference server deployment"
|
||||
|
||||
# Verify compose file exists
|
||||
if [[ ! -f "${COMPOSE_FILE}" ]]; then
|
||||
log "ERROR: Compose file not found at ${COMPOSE_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "${PROJECT_DIR}"
|
||||
|
||||
# Pull latest compose file (in case it was updated)
|
||||
log "Pulling latest compose file from git"
|
||||
git pull --quiet origin main || log "WARNING: git pull failed, using local compose file"
|
||||
|
||||
# Pull latest image
|
||||
# ECR_PUBLIC_REGISTRY_ALIAS comes from webhook env payload
|
||||
log "Pulling latest image: public.ecr.aws/${ECR_PUBLIC_REGISTRY_ALIAS}/msk-cv-inference-server:latest"
|
||||
docker compose -f "${COMPOSE_FILE}" pull cv-inference-server
|
||||
|
||||
# Deploy with zero-downtime recreation
|
||||
# All required env vars (TRITON_ENDPOINT, CV_INFERENCE_HOST, etc.) are already
|
||||
# in the shell environment, passed from webhook payload
|
||||
log "Recreating cv-inference-server container"
|
||||
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate cv-inference-server
|
||||
|
||||
# Clean up old images
|
||||
log "Pruning unused images"
|
||||
docker image prune -f
|
||||
|
||||
# Verify health
|
||||
log "Waiting for health check..."
|
||||
for i in {1..30}; do
|
||||
if docker compose -f "${COMPOSE_FILE}" exec -T cv-inference-server \
|
||||
python -c "import urllib.request; urllib.request.urlopen('http://localhost:8001/health')" 2>/dev/null; then
|
||||
log "Health check passed"
|
||||
break
|
||||
fi
|
||||
if [[ $i -eq 30 ]]; then
|
||||
log "ERROR: Health check failed after 30 attempts"
|
||||
docker compose -f "${COMPOSE_FILE}" logs cv-inference-server
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Deployment completed successfully"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,18 @@
|
||||
# /opt/pilot-project/Caddyfile
|
||||
# Replace app.example.com with your actual domain
|
||||
|
||||
app.example.com {
|
||||
reverse_proxy lumina-frontend:80
|
||||
|
||||
header {
|
||||
Strict-Transport-Security "max-age=31536000; includeSubDomains"
|
||||
X-Content-Type-Options "nosniff"
|
||||
X-Frame-Options "SAMEORIGIN"
|
||||
Referrer-Policy "strict-origin-when-cross-origin"
|
||||
}
|
||||
|
||||
@static {
|
||||
path *.js *.css *.wasm *.png *.jpg *.svg *.woff2 *.ico *.map
|
||||
}
|
||||
header @static Cache-Control "public, max-age=31536000, immutable"
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
lumina-frontend:
|
||||
image: public.ecr.aws/${ECR_PUBLIC_REGISTRY_ALIAS}/lumina-frontend:latest
|
||||
container_name: lumina-frontend
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:80"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 128m
|
||||
cpus: '0.5'
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/health"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
networks:
|
||||
default:
|
||||
name: pilot-network
|
||||
external: true
|
||||
@@ -0,0 +1,14 @@
|
||||
# Lightsail Production Environment
|
||||
# Copy to /opt/pilot-project/.env on Lightsail VM
|
||||
# chmod 600 /opt/pilot-project/.env
|
||||
|
||||
# ECR Public Registry Alias (e.g., vkist-pilot, your-account-alias)
|
||||
# Get from: AWS Console > ECR Public > Registries > [your registry] > Alias
|
||||
ECR_PUBLIC_REGISTRY_ALIAS=vkist-pilot
|
||||
|
||||
# Webhook Secret (generate with: openssl rand -hex 32)
|
||||
# Must match Gitea webhook secret configuration
|
||||
WEBHOOK_SECRET=your-generated-secret-here
|
||||
|
||||
# Optional: Backend API URL for nginx proxy (if different from frontend)
|
||||
# BACKEND_API_URL=http://localhost:8001
|
||||
@@ -0,0 +1,26 @@
|
||||
[Unit]
|
||||
Description=Gitea Webhook Listener for Lumina Frontend Deploy
|
||||
After=network.target docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ubuntu
|
||||
Group=ubuntu
|
||||
WorkingDirectory=/opt/pilot-project
|
||||
EnvironmentFile=/opt/pilot-project/.env
|
||||
ExecStart=/usr/bin/python3 /opt/pilot-project/webhook-listener.py
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/opt/pilot-project
|
||||
ProtectHome=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# Deploy script for Lumina MSK Frontend
|
||||
# Triggered by Gitea webhook - receives env vars from Gitea repo variables
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
PROJECT_DIR="/opt/pilot-project"
|
||||
COMPOSE_FILE="${PROJECT_DIR}/docker-compose.frontend.yaml"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
}
|
||||
|
||||
# Required env vars (passed from webhook / Gitea variables)
|
||||
REQUIRED_VARS=(
|
||||
"ECR_PUBLIC_REGISTRY_ALIAS"
|
||||
"CORS_ORIGINS"
|
||||
)
|
||||
|
||||
# Write .env from passed environment variables (so docker compose env_file works)
|
||||
write_env_file() {
|
||||
local env_file="${PROJECT_DIR}/.env"
|
||||
log "Writing .env from passed environment variables"
|
||||
for var in "${REQUIRED_VARS[@]}"; do
|
||||
if [[ -n "${!var:-}" ]]; then
|
||||
echo "${var}=${!var}" >> "${env_file}"
|
||||
else
|
||||
log "WARNING: ${var} not set in environment"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
log "Starting frontend deployment"
|
||||
|
||||
# Verify compose file exists
|
||||
if [[ ! -f "${COMPOSE_FILE}" ]]; then
|
||||
log "ERROR: Compose file not found at ${COMPOSE_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "${PROJECT_DIR}"
|
||||
|
||||
# Pull latest compose file (in case it was updated)
|
||||
log "Pulling latest compose file from git"
|
||||
git pull --quiet origin main || log "WARNING: git pull failed, using local compose file"
|
||||
|
||||
# Write .env from passed env vars
|
||||
write_env_file
|
||||
|
||||
# Pull latest image
|
||||
log "Pulling latest image: public.ecr.aws/${ECR_PUBLIC_REGISTRY_ALIAS}/lumina-frontend:latest"
|
||||
docker compose -f "${COMPOSE_FILE}" pull lumina-frontend
|
||||
|
||||
# Deploy with zero-downtime recreation
|
||||
log "Recreating frontend container"
|
||||
docker compose -f "${COMPOSE_FILE}" up -d --force-recreate lumina-frontend
|
||||
|
||||
# Clean up old images
|
||||
log "Pruning unused images"
|
||||
docker image prune -f
|
||||
|
||||
# Verify health
|
||||
log "Waiting for health check..."
|
||||
for i in {1..30}; do
|
||||
if docker compose -f "${COMPOSE_FILE}" exec -T lumina-frontend wget --quiet --tries=1 --spider http://localhost/ 2>/dev/null; then
|
||||
log "Health check passed"
|
||||
break
|
||||
fi
|
||||
if [[ $i -eq 30 ]]; then
|
||||
log "ERROR: Health check failed after 30 attempts"
|
||||
docker compose -f "${COMPOSE_FILE}" logs lumina-frontend
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
log "Deployment completed successfully"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Gitea Webhook Listener for Lumina Deployment
|
||||
Runs on Lightsail VM, receives webhook from Gitea, triggers the appropriate
|
||||
deploy script based on which workflow completed.
|
||||
"""
|
||||
|
||||
import os
|
||||
import hmac
|
||||
import hashlib
|
||||
import subprocess
|
||||
import logging
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
||||
# Configuration
|
||||
WEBHOOK_SECRET = os.environ.get('WEBHOOK_SECRET')
|
||||
DEPLOY_SCRIPTS = {
|
||||
'Frontend CI/CD': '/opt/pilot-project/webhook-deploy.sh',
|
||||
'Backend ECR Deployment': '/opt/pilot-project/webhook-deploy-backend.sh',
|
||||
'Backend Modal Build': '/opt/pilot-project/webhook-deploy-backend.sh',
|
||||
}
|
||||
LISTEN_HOST = '127.0.0.1'
|
||||
LISTEN_PORT = 3333
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WebhookHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
# Verify path
|
||||
if self.path != '/deploy':
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# Read body
|
||||
content_length = int(self.headers.get('Content-Length', 0))
|
||||
body = self.rfile.read(content_length)
|
||||
|
||||
# Verify signature
|
||||
signature = self.headers.get('X-Gitea-Signature', '')
|
||||
if not self.verify_signature(body, signature):
|
||||
logger.warning("Invalid webhook signature")
|
||||
self.send_response(401)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'Invalid signature')
|
||||
return
|
||||
|
||||
# Parse JSON payload
|
||||
try:
|
||||
import json
|
||||
payload = json.loads(body.decode('utf-8'))
|
||||
except json.JSONDecodeError:
|
||||
logger.error("Invalid JSON payload")
|
||||
self.send_response(400)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
# Check if workflow completed successfully
|
||||
deploy_info = self.should_deploy(payload)
|
||||
if not deploy_info:
|
||||
logger.info("Event does not trigger deploy: %s", payload.get('action', 'unknown'))
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'Event ignored')
|
||||
return
|
||||
|
||||
deploy_script = deploy_info['script']
|
||||
deploy_env = deploy_info.get('env', {})
|
||||
|
||||
# Trigger deployment with env vars from payload
|
||||
logger.info("Triggering deployment with script: %s", deploy_script)
|
||||
logger.debug("Passing env vars: %s", list(deploy_env.keys()))
|
||||
try:
|
||||
# Merge payload env vars with current process env
|
||||
proc_env = os.environ.copy()
|
||||
proc_env.update(deploy_env)
|
||||
|
||||
result = subprocess.run(
|
||||
[deploy_script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
env=proc_env
|
||||
)
|
||||
if result.returncode == 0:
|
||||
logger.info("Deployment successful")
|
||||
logger.debug("Deploy output: %s", result.stdout)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'Deployment triggered successfully')
|
||||
else:
|
||||
logger.error("Deployment failed: %s", result.stderr)
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
self.wfile.write(f'Deployment failed: {result.stderr}'.encode())
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error("Deployment timed out")
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'Deployment timed out')
|
||||
except Exception as e:
|
||||
logger.exception("Deployment error")
|
||||
self.send_response(500)
|
||||
self.end_headers()
|
||||
self.wfile.write(f'Deployment error: {str(e)}'.encode())
|
||||
|
||||
def verify_signature(self, body: bytes, signature: str) -> bool:
|
||||
"""Verify Gitea HMAC-SHA256 signature"""
|
||||
if not WEBHOOK_SECRET:
|
||||
logger.error("WEBHOOK_SECRET not configured")
|
||||
return False
|
||||
expected = 'sha256=' + hmac.new(
|
||||
WEBHOOK_SECRET.encode(),
|
||||
body,
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
return hmac.compare_digest(expected, signature)
|
||||
|
||||
def should_deploy(self, payload: dict) -> dict | None:
|
||||
"""Determine if this webhook event should trigger a deploy.
|
||||
Returns dict with 'script' and optional 'env' if yes, None if no."""
|
||||
# Gitea workflow_run event
|
||||
if payload.get('action') == 'completed':
|
||||
workflow = payload.get('workflow', {})
|
||||
workflow_name = workflow.get('name', '')
|
||||
conclusion = payload.get('conclusion')
|
||||
if conclusion == 'success' and workflow_name in DEPLOY_SCRIPTS:
|
||||
return {
|
||||
'script': DEPLOY_SCRIPTS[workflow_name],
|
||||
'env': payload.get('env', {})
|
||||
}
|
||||
# Direct webhook payload (for manual triggers or extended format)
|
||||
if 'service' in payload:
|
||||
service = payload.get('service')
|
||||
if service == 'cv-inference-server':
|
||||
return {
|
||||
'script': DEPLOY_SCRIPTS['Backend ECR Deployment'],
|
||||
'env': payload.get('env', {})
|
||||
}
|
||||
if service == 'lumina-frontend':
|
||||
return {
|
||||
'script': DEPLOY_SCRIPTS['Frontend CI/CD'],
|
||||
'env': payload.get('env', {})
|
||||
}
|
||||
return None
|
||||
|
||||
def log_message(self, format, *args):
|
||||
logger.info("%s - %s", self.address_string(), format % args)
|
||||
|
||||
|
||||
def main():
|
||||
if not WEBHOOK_SECRET:
|
||||
logger.error("WEBHOOK_SECRET environment variable not set")
|
||||
exit(1)
|
||||
|
||||
# Verify deploy scripts exist
|
||||
for name, path in DEPLOY_SCRIPTS.items():
|
||||
if not os.path.exists(path):
|
||||
logger.warning("Deploy script not found for %s at %s", name, path)
|
||||
|
||||
server = HTTPServer((LISTEN_HOST, LISTEN_PORT), WebhookHandler)
|
||||
logger.info("Webhook listener starting on %s:%s", LISTEN_HOST, LISTEN_PORT)
|
||||
logger.info("Watching workflows: %s", ', '.join(DEPLOY_SCRIPTS.keys()))
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down")
|
||||
server.shutdown()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,33 +0,0 @@
|
||||
import modal
|
||||
import subprocess
|
||||
|
||||
# 1. Define the App
|
||||
app = modal.App("hello-world-build-worker")
|
||||
|
||||
# 2. Define a clean container image representing your worker environment
|
||||
image = modal.Image.debian_slim().pip_install("requests")
|
||||
|
||||
# 3. Define the worker function
|
||||
@app.function(
|
||||
image=image,
|
||||
# This keeps the worker container alive for 5 minutes after finishing,
|
||||
# so subsequent runs start instantly (avoiding cold starts).
|
||||
scaledown_window=300
|
||||
)
|
||||
def run_worker_job():
|
||||
print("--- WORKER CONTAINER STARTED ---")
|
||||
|
||||
# Mimic fetching project files or setting up environment variables
|
||||
test_env_var = "Lumina-Build-Sandbox"
|
||||
print(f"Environment initialized. Target deployment: {test_env_var}")
|
||||
|
||||
# Run a simple shell command inside the container to mimic building/testing
|
||||
print("Running build script simulation...")
|
||||
result = subprocess.run(
|
||||
["echo", "Hello World! Your Modal container is successfully building your Python app."],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
|
||||
print(f"Build output: {result.stdout.strip()}")
|
||||
print("--- WORKER JOB COMPLETED SUCCESSFULLY ---")
|
||||
Reference in New Issue
Block a user