gitea_modal_build update
This commit is contained in:
@@ -22,64 +22,30 @@ import json
|
|||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
import modal
|
import modal
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Configuration - Dynamic paths relative to this script
|
# Configuration
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
# Get the directory where this script actually lives on the machine running it
|
|
||||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
||||||
|
|
||||||
# When running inside Modal, the project files are baked into /workspace.
|
|
||||||
# When running locally, walk up from the script location.
|
|
||||||
_MODAL_WORKSPACE = Path("/workspace")
|
|
||||||
if (_MODAL_WORKSPACE / "backend").exists():
|
|
||||||
PROJECT_ROOT = _MODAL_WORKSPACE
|
|
||||||
DOCKERFILE_PATH = _MODAL_WORKSPACE / "deps" / "implementation" / "backend_deploy" / "Dockerfile"
|
|
||||||
else:
|
|
||||||
PROJECT_ROOT = SCRIPT_DIR
|
|
||||||
while not (PROJECT_ROOT / "backend").exists() and PROJECT_ROOT != PROJECT_ROOT.parent:
|
|
||||||
PROJECT_ROOT = PROJECT_ROOT.parent
|
|
||||||
DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile"
|
|
||||||
|
|
||||||
print(f"--- Environment Verification ---")
|
|
||||||
print(f"Script location: {SCRIPT_DIR}")
|
|
||||||
print(f"Resolved PROJECT_ROOT: {PROJECT_ROOT} (Exists: {PROJECT_ROOT.exists()})")
|
|
||||||
print(f"Resolved DOCKERFILE_PATH: {DOCKERFILE_PATH} (Exists: {DOCKERFILE_PATH.exists()})")
|
|
||||||
print(f"----------------------------------------")
|
|
||||||
|
|
||||||
# Project layout variables
|
|
||||||
BACKEND_SOURCE = "backend"
|
|
||||||
REQUIREMENTS_FILE = "requirements.txt"
|
|
||||||
|
|
||||||
# Default registry/image settings
|
|
||||||
DEFAULT_REGISTRY = "public.ecr.aws/i9a4e3f6/msk-cv-inference-server"
|
DEFAULT_REGISTRY = "public.ecr.aws/i9a4e3f6/msk-cv-inference-server"
|
||||||
DEFAULT_IMAGE_NAME = "msk-lumina-backend"
|
DEFAULT_IMAGE_NAME = "msk-lumina-backend"
|
||||||
DEFAULT_TAG = "latest"
|
DEFAULT_TAG = "latest"
|
||||||
|
|
||||||
# Modal app configuration
|
|
||||||
APP_NAME = "msk-lumina-backend-builder"
|
APP_NAME = "msk-lumina-backend-builder"
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Modal Image Definition - Kaniko-based builder (no Docker daemon required)
|
# Modal Image Definition
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
# Install Kaniko from chainguard-forks and AWS auth dependencies.
|
|
||||||
# The project source files are baked into the image at /workspace so Kaniko
|
|
||||||
# can access the build context and Dockerfile at runtime.
|
|
||||||
builder_image = (
|
builder_image = (
|
||||||
modal.Image.debian_slim()
|
# 1. Start from Kaniko debug (includes a basic shell, necessary for Modal)
|
||||||
|
modal.Image.from_registry("gcr.io/kaniko-project/executor:debug")
|
||||||
|
|
||||||
|
# 2. Inject python-pip and dependencies natively on top of the image
|
||||||
.apt_install("curl", "ca-certificates")
|
.apt_install("curl", "ca-certificates")
|
||||||
.run_commands(
|
.pip_install("boto3")
|
||||||
"curl -sSL https://github.com/chainguard-forks/kaniko/releases/latest/download/kaniko -o /usr/local/bin/kaniko",
|
|
||||||
"chmod +x /usr/local/bin/kaniko",
|
# 3. Mount your local project directory context to /workspace
|
||||||
)
|
|
||||||
.pip_install("boto3") # for ECR auth token retrieval
|
|
||||||
.add_local_dir(
|
.add_local_dir(
|
||||||
str(PROJECT_ROOT),
|
os.getcwd(), # Or your dynamic project root lookup path
|
||||||
remote_path="/workspace",
|
remote_path="/workspace",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -89,14 +55,11 @@ app = modal.App(APP_NAME, image=builder_image)
|
|||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Build and Push Function (Kaniko)
|
# Build and Push Function (Kaniko)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
@app.function(
|
@app.function(
|
||||||
timeout=900, # 15 min for heavy ML deps (torch, transformers, etc.)
|
timeout=900,
|
||||||
cpu=4,
|
cpu=4,
|
||||||
memory=8192, # 8GB RAM - plenty for wheel building
|
memory=8192,
|
||||||
secrets=[
|
secrets=[modal.Secret.from_name("aws-secrets")],
|
||||||
modal.Secret.from_name("aws-secrets"), # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
def build_and_push(
|
def build_and_push(
|
||||||
registry: str = DEFAULT_REGISTRY,
|
registry: str = DEFAULT_REGISTRY,
|
||||||
@@ -105,23 +68,15 @@ def build_and_push(
|
|||||||
push: bool = True,
|
push: bool = True,
|
||||||
platform: str = "linux/amd64",
|
platform: str = "linux/amd64",
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""
|
|
||||||
Build the backend Docker image with Kaniko and push to registry.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
registry: Container registry hostname (e.g., public.ecr.aws/vkist-project)
|
|
||||||
image_name: Image name (e.g., vkist-backend)
|
|
||||||
tag: Image tag (e.g., v1.2.3, latest, git-sha)
|
|
||||||
push: Whether to push to registry
|
|
||||||
platform: Target platform (linux/amd64, linux/arm64)
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with image reference and build status
|
|
||||||
"""
|
|
||||||
import boto3
|
import boto3
|
||||||
full_image = f"{registry}/{image_name}:{tag}"
|
full_image = f"{registry}/{image_name}:{tag}"
|
||||||
print(f"Building {full_image} for {platform}")
|
print(f"Building {full_image} for {platform}")
|
||||||
|
|
||||||
|
# Explicitly use the inside-container paths
|
||||||
|
workspace_root = Path("/workspace")
|
||||||
|
# Adjust this path if your Dockerfile lives in a specific subdirectory inside your workspace
|
||||||
|
dockerfile_path = workspace_root / "Dockerfile"
|
||||||
|
|
||||||
# Prepare docker config for Kaniko if pushing to ECR
|
# Prepare docker config for Kaniko if pushing to ECR
|
||||||
docker_config_dir = None
|
docker_config_dir = None
|
||||||
if push and ("ecr.aws" in registry or "ecr." in registry):
|
if push and ("ecr.aws" in registry or "ecr." in registry):
|
||||||
@@ -142,16 +97,16 @@ def build_and_push(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
docker_config_dir = PROJECT_ROOT / ".kaniko"
|
docker_config_dir = workspace_root / ".kaniko"
|
||||||
docker_config_dir.mkdir(exist_ok=True)
|
docker_config_dir.mkdir(exist_ok=True)
|
||||||
(docker_config_dir / "config.json").write_text(json.dumps(config))
|
(docker_config_dir / "config.json").write_text(json.dumps(config))
|
||||||
print(f"ECR auth configured for {registry}")
|
print(f"ECR auth configured for {registry}")
|
||||||
|
|
||||||
# Build with Kaniko
|
# Build with Kaniko
|
||||||
kaniko_cmd = [
|
kaniko_cmd = [
|
||||||
"/usr/local/bin/kaniko",
|
"/kaniko/executor", # Correct path inside the official debug image
|
||||||
"--context", str(PROJECT_ROOT),
|
"--context", str(workspace_root),
|
||||||
"--dockerfile", str(DOCKERFILE_PATH),
|
"--dockerfile", str(dockerfile_path),
|
||||||
"--destination", full_image,
|
"--destination", full_image,
|
||||||
"--platform", platform,
|
"--platform", platform,
|
||||||
"--verbosity", "info",
|
"--verbosity", "info",
|
||||||
|
|||||||
Reference in New Issue
Block a user