update the gitea
Some checks failed
Backend EC - Modal Deployment / build-and-push (push) Failing after 28s
Backend EC - Modal Deployment / notify-deploy (push) Has been skipped

This commit is contained in:
DatTT127
2026-07-18 22:55:49 +07:00
parent 9b43474991
commit ccb5d4c82d

View File

@@ -1,6 +1,9 @@
""" """
Modal script to build and push VKIST backend Docker image to container registry. Modal script to build and push VKIST backend Docker image to container registry.
Uses Kaniko (daemonless Docker builder) inside Modal to build and push
without requiring a local Docker daemon.
Run from Gitea runner on Lightsail (2GB RAM) - offloads heavy build to Modal serverless. Run from Gitea runner on Lightsail (2GB RAM) - offloads heavy build to Modal serverless.
Usage: Usage:
@@ -14,8 +17,13 @@ Usage:
modal deploy gitea_modal_build.py modal deploy gitea_modal_build.py
""" """
import base64
import json
import os import os
from pathlib import Path from pathlib import Path
import subprocess
import boto3
import modal import modal
# ============================================================================= # =============================================================================
@@ -25,13 +33,16 @@ import modal
# Get the directory where this script actually lives on the machine running it # Get the directory where this script actually lives on the machine running it
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
# Walk up until we find a marker directory that identifies the repo root. # When running inside Modal, the project files are baked into /workspace.
# This makes the script robust regardless of where Modal mounts it. # When running locally, walk up from the script location.
_MODAL_WORKSPACE = Path("/workspace")
if (_MODAL_WORKSPACE / "backend").exists():
PROJECT_ROOT = _MODAL_WORKSPACE
DOCKERFILE_PATH = _MODAL_WORKSPACE / "deps" / "implementation" / "backend_deploy" / "Dockerfile"
else:
PROJECT_ROOT = SCRIPT_DIR PROJECT_ROOT = SCRIPT_DIR
while not (PROJECT_ROOT / "backend").exists() and PROJECT_ROOT != PROJECT_ROOT.parent: while not (PROJECT_ROOT / "backend").exists() and PROJECT_ROOT != PROJECT_ROOT.parent:
PROJECT_ROOT = PROJECT_ROOT.parent PROJECT_ROOT = PROJECT_ROOT.parent
# Explicitly find your local Dockerfile
DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile" DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile"
print(f"--- Environment Verification ---") print(f"--- Environment Verification ---")
@@ -53,23 +64,30 @@ DEFAULT_TAG = "latest"
APP_NAME = "msk-lumina-backend-builder" APP_NAME = "msk-lumina-backend-builder"
# ============================================================================= # =============================================================================
# Modal Image Definition - uses the project's Dockerfile for consistency # Modal Image Definition - Kaniko-based builder (no Docker daemon required)
# ============================================================================= # =============================================================================
# Build the image using the existing multi-stage Dockerfile # Install Kaniko from chainguard-forks and AWS auth dependencies.
# We set context_dir to PROJECT_ROOT so the Dockerfile can see ../../../../backend # The project source files are baked into the image at /workspace so Kaniko
backend_image = modal.Image.from_dockerfile( # can access the build context and Dockerfile at runtime.
str(DOCKERFILE_PATH), builder_image = (
context_dir=str(PROJECT_ROOT), modal.Image.debian_slim()
).pip_install( .apt_install("curl", "ca-certificates")
"docker", # for test_image function .run_commands(
"boto3", # for ECR auth if needed "curl -sSL https://github.com/chainguard-forks/kaniko/releases/latest/download/kaniko -o /usr/local/bin/kaniko",
"chmod +x /usr/local/bin/kaniko",
)
.pip_install("boto3") # for ECR auth token retrieval
.add_local_dir(
str(PROJECT_ROOT),
remote_path="/workspace",
)
) )
app = modal.App(APP_NAME, image=backend_image) app = modal.App(APP_NAME, image=builder_image)
# ============================================================================= # =============================================================================
# Build and Push Function # Build and Push Function (Kaniko)
# ============================================================================= # =============================================================================
@app.function( @app.function(
@@ -88,7 +106,7 @@ def build_and_push(
platform: str = "linux/amd64", platform: str = "linux/amd64",
) -> dict: ) -> dict:
""" """
Build the backend Docker image and push to registry. Build the backend Docker image with Kaniko and push to registry.
Args: Args:
registry: Container registry hostname (e.g., public.ecr.aws/vkist-project) registry: Container registry hostname (e.g., public.ecr.aws/vkist-project)
@@ -98,128 +116,63 @@ def build_and_push(
platform: Target platform (linux/amd64, linux/arm64) platform: Target platform (linux/amd64, linux/arm64)
Returns: Returns:
Dict with image reference, digest, and size info Dict with image reference and build status
""" """
import docker
full_image = f"{registry}/{image_name}:{tag}" full_image = f"{registry}/{image_name}:{tag}"
print(f"Building {full_image} for {platform}") print(f"Building {full_image} for {platform}")
# Initialize Docker client (uses Modal's container runtime) # Prepare docker config for Kaniko if pushing to ECR
client = docker.from_env() docker_config_dir = None
if push and ("ecr.aws" in registry or "ecr." in registry):
print(f"Configuring ECR authentication...")
region = os.getenv("AWS_REGION", "us-east-1")
# Build using the Dockerfile from project root context ecr = boto3.client("ecr", region_name=region)
print(f"Build context: {PROJECT_ROOT}") auth = ecr.get_authorization_token()
print(f"Dockerfile: {DOCKERFILE_PATH}") token = auth["authorizationData"][0]["authorizationToken"]
username, password = base64.b64decode(token).decode().split(":")
# Build the image config = {
image, build_logs = client.images.build( "auths": {
path=PROJECT_ROOT, registry: {
dockerfile=DOCKERFILE_PATH, "username": username,
tag=full_image, "password": password,
platform=platform, }
rm=True, }
pull=True,
buildargs={
"BUILDKIT_INLINE_CACHE": "1",
},
)
# Stream build logs
for chunk in build_logs:
if "stream" in chunk:
print(chunk["stream"].strip())
elif "error" in chunk:
raise RuntimeError(f"Build failed: {chunk['error']}")
print(f"Build complete: {image.id[:12]}")
# Get image details
image_info = client.images.get(full_image)
size_bytes = image_info.attrs["Size"]
digest = image_info.attrs.get("RepoDigests", ["unknown"])[0]
result = {
"image": full_image,
"digest": digest,
"size_bytes": size_bytes,
"image_id": image.id,
} }
if push: docker_config_dir = PROJECT_ROOT / ".kaniko"
print(f"Pushing to {registry}...") docker_config_dir.mkdir(exist_ok=True)
(docker_config_dir / "config.json").write_text(json.dumps(config))
print(f"ECR auth configured for {registry}")
# Authenticate with registry based on hostname # Build with Kaniko
_authenticate_registry(registry, client) kaniko_cmd = [
"/usr/local/bin/kaniko",
"--context", str(PROJECT_ROOT),
"--dockerfile", str(DOCKERFILE_PATH),
"--destination", full_image,
"--platform", platform,
"--verbosity", "info",
]
# Push image if docker_config_dir:
push_logs = client.images.push( kaniko_cmd.extend(["--docker-config", str(docker_config_dir)])
repository=f"{registry}/{image_name}",
tag=tag,
stream=True,
decode=True,
)
for chunk in push_logs: print(f"Running Kaniko: {' '.join(kaniko_cmd)}")
if "status" in chunk: result = subprocess.run(kaniko_cmd, capture_output=True, text=True)
print(f" {chunk['status']}: {chunk.get('progress', '')}")
if "error" in chunk: print(result.stdout)
raise RuntimeError(f"Push failed: {chunk['error']}") if result.returncode != 0:
raise RuntimeError(f"Kaniko build failed:\n{result.stderr}")
print(f"Push complete: {full_image}") print(f"Push complete: {full_image}")
# Get digest after push
pushed_image = client.images.get(full_image)
result["digest"] = pushed_image.attrs.get("RepoDigests", ["unknown"])[0]
return result
def _authenticate_registry(registry: str, client):
"""Authenticate Docker client with the target registry."""
import subprocess
if "ecr.aws" in registry or "ecr." in registry:
# AWS ECR (public or private)
region = os.getenv("AWS_REGION", "us-east-1")
print(f"Authenticating with AWS ECR ({region})...")
subprocess.run([
"aws", "ecr", "get-login-password", "--region", region
], check=True, capture_output=True)
# Note: Modal's aws-ecr-credentials secret should handle this via env vars
@app.function(
image=backend_image,
timeout=120,
secrets=[modal.Secret.from_name("aws-secrets")],
)
def test_image(registry: str, image_name: str, tag: str) -> dict:
"""Test the built image by running a quick health check."""
import docker
full_image = f"{registry}/{image_name}:{tag}"
client = docker.from_env()
print(f"Pulling {full_image} for testing...")
client.images.pull(full_image)
print("Running basic smoke test...")
output = client.containers.run(
full_image,
command=[
"python", "-c",
"import sys; print('Python', sys.version); "
"import cv2; print('OpenCV', cv2.__version__); "
],
remove=True,
detach=False,
)
return { return {
"status": "healthy",
"output": output.decode().strip(),
"image": full_image, "image": full_image,
"digest": "unknown",
"size_bytes": 0,
"image_id": "kaniko-build",
} }
@@ -237,7 +190,7 @@ def main(
test: bool = False, test: bool = False,
): ):
""" """
Build and push backend Docker image via Modal. Build and push backend Docker image via Kaniko on Modal.
Args: Args:
registry: Container registry (default: public.ecr.aws/vkist-project) registry: Container registry (default: public.ecr.aws/vkist-project)
@@ -245,10 +198,13 @@ def main(
tag: Image tag (default: latest) tag: Image tag (default: latest)
push: Push to registry (default: true) push: Push to registry (default: true)
platform: Target platform (default: linux/amd64) platform: Target platform (default: linux/amd64)
test: Run health check after build (default: false) test: Run health check after build (default: false, unsupported with Kaniko)
""" """
if test:
print("WARNING: post-push testing is not supported with Kaniko. Skipping test.")
print("=" * 60) print("=" * 60)
print("MSK-Lumina Backend Docker Build on Modal") print("MSK-Lumina Backend Docker Build on Modal (Kaniko)")
print("=" * 60) print("=" * 60)
print(f"Registry: {registry}") print(f"Registry: {registry}")
print(f"Image: {image_name}") print(f"Image: {image_name}")
@@ -258,7 +214,6 @@ def main(
print(f"Test: {test}") print(f"Test: {test}")
print("=" * 60) print("=" * 60)
# Build and push
result = build_and_push.remote( result = build_and_push.remote(
registry=registry, registry=registry,
image_name=image_name, image_name=image_name,
@@ -272,7 +227,6 @@ def main(
print("=" * 60) print("=" * 60)
print(f"Image: {result['image']}") print(f"Image: {result['image']}")
print(f"Digest: {result['digest']}") print(f"Digest: {result['digest']}")
print(f"Size: {result['size_bytes'] / 1024 / 1024:.1f} MB")
print("=" * 60) print("=" * 60)
# Output for CI/CD parsing (GitHub Actions / Gitea Actions format) # Output for CI/CD parsing (GitHub Actions / Gitea Actions format)
@@ -280,13 +234,6 @@ def main(
print(f"::set-output name=digest::{result['digest']}") print(f"::set-output name=digest::{result['digest']}")
print(f"::set-output name=size_mb::{result['size_bytes'] / 1024 / 1024:.1f}") print(f"::set-output name=size_mb::{result['size_bytes'] / 1024 / 1024:.1f}")
# Optional test
if test and push:
print("\nRunning health check...")
test_result = test_image.remote(registry, image_name, tag)
print(f"Health check: {test_result['status']}")
print(f"Output: {test_result['output']}")
if __name__ == "__main__": if __name__ == "__main__":
# Allow direct execution for debugging # Allow direct execution for debugging