update the gitea
Some checks failed
Backend EC - Modal Deployment / build-and-push (push) Failing after 28s
Backend EC - Modal Deployment / notify-deploy (push) Has been skipped

This commit is contained in:
DatTT127
2026-07-18 22:55:49 +07:00
parent 9b43474991
commit ccb5d4c82d

View File

@@ -1,6 +1,9 @@
""" """
Modal script to build and push VKIST backend Docker image to container registry. Modal script to build and push VKIST backend Docker image to container registry.
Uses Kaniko (daemonless Docker builder) inside Modal to build and push
without requiring a local Docker daemon.
Run from Gitea runner on Lightsail (2GB RAM) - offloads heavy build to Modal serverless. Run from Gitea runner on Lightsail (2GB RAM) - offloads heavy build to Modal serverless.
Usage: Usage:
@@ -14,8 +17,13 @@ Usage:
modal deploy gitea_modal_build.py modal deploy gitea_modal_build.py
""" """
import base64
import json
import os import os
from pathlib import Path from pathlib import Path
import subprocess
import boto3
import modal import modal
# ============================================================================= # =============================================================================
@@ -25,14 +33,17 @@ import modal
# Get the directory where this script actually lives on the machine running it # Get the directory where this script actually lives on the machine running it
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
# Walk up until we find a marker directory that identifies the repo root. # When running inside Modal, the project files are baked into /workspace.
# This makes the script robust regardless of where Modal mounts it. # When running locally, walk up from the script location.
PROJECT_ROOT = SCRIPT_DIR _MODAL_WORKSPACE = Path("/workspace")
while not (PROJECT_ROOT / "backend").exists() and PROJECT_ROOT != PROJECT_ROOT.parent: if (_MODAL_WORKSPACE / "backend").exists():
PROJECT_ROOT = PROJECT_ROOT.parent PROJECT_ROOT = _MODAL_WORKSPACE
DOCKERFILE_PATH = _MODAL_WORKSPACE / "deps" / "implementation" / "backend_deploy" / "Dockerfile"
# Explicitly find your local Dockerfile else:
DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile" PROJECT_ROOT = SCRIPT_DIR
while not (PROJECT_ROOT / "backend").exists() and PROJECT_ROOT != PROJECT_ROOT.parent:
PROJECT_ROOT = PROJECT_ROOT.parent
DOCKERFILE_PATH = SCRIPT_DIR / "Dockerfile"
print(f"--- Environment Verification ---") print(f"--- Environment Verification ---")
print(f"Script location: {SCRIPT_DIR}") print(f"Script location: {SCRIPT_DIR}")
@@ -53,23 +64,30 @@ DEFAULT_TAG = "latest"
APP_NAME = "msk-lumina-backend-builder" APP_NAME = "msk-lumina-backend-builder"
# ============================================================================= # =============================================================================
# Modal Image Definition - uses the project's Dockerfile for consistency # Modal Image Definition - Kaniko-based builder (no Docker daemon required)
# ============================================================================= # =============================================================================
# Build the image using the existing multi-stage Dockerfile # Install Kaniko from chainguard-forks and AWS auth dependencies.
# We set context_dir to PROJECT_ROOT so the Dockerfile can see ../../../../backend # The project source files are baked into the image at /workspace so Kaniko
backend_image = modal.Image.from_dockerfile( # can access the build context and Dockerfile at runtime.
str(DOCKERFILE_PATH), builder_image = (
context_dir=str(PROJECT_ROOT), modal.Image.debian_slim()
).pip_install( .apt_install("curl", "ca-certificates")
"docker", # for test_image function .run_commands(
"boto3", # for ECR auth if needed "curl -sSL https://github.com/chainguard-forks/kaniko/releases/latest/download/kaniko -o /usr/local/bin/kaniko",
"chmod +x /usr/local/bin/kaniko",
)
.pip_install("boto3") # for ECR auth token retrieval
.add_local_dir(
str(PROJECT_ROOT),
remote_path="/workspace",
)
) )
app = modal.App(APP_NAME, image=backend_image) app = modal.App(APP_NAME, image=builder_image)
# ============================================================================= # =============================================================================
# Build and Push Function # Build and Push Function (Kaniko)
# ============================================================================= # =============================================================================
@app.function( @app.function(
@@ -88,138 +106,73 @@ def build_and_push(
platform: str = "linux/amd64", platform: str = "linux/amd64",
) -> dict: ) -> dict:
""" """
Build the backend Docker image and push to registry. Build the backend Docker image with Kaniko and push to registry.
Args: Args:
registry: Container registry hostname (e.g., public.ecr.aws/vkist-project) registry: Container registry hostname (e.g., public.ecr.aws/vkist-project)
image_name: Image name (e.g., vkist-backend) image_name: Image name (e.g., vkist-backend)
tag: Image tag (e.g., v1.2.3, latest, git-sha) tag: Image tag (e.g., v1.2.3, latest, git-sha)
push: Whether to push to registry push: Whether to push to registry
platform: Target platform (linux/amd64, linux/arm64) platform: Target platform (linux/amd64, linux/arm64)
Returns: Returns:
Dict with image reference, digest, and size info Dict with image reference and build status
""" """
import docker
full_image = f"{registry}/{image_name}:{tag}" full_image = f"{registry}/{image_name}:{tag}"
print(f"Building {full_image} for {platform}") print(f"Building {full_image} for {platform}")
# Initialize Docker client (uses Modal's container runtime)
client = docker.from_env()
# Build using the Dockerfile from project root context
print(f"Build context: {PROJECT_ROOT}")
print(f"Dockerfile: {DOCKERFILE_PATH}")
# Build the image
image, build_logs = client.images.build(
path=PROJECT_ROOT,
dockerfile=DOCKERFILE_PATH,
tag=full_image,
platform=platform,
rm=True,
pull=True,
buildargs={
"BUILDKIT_INLINE_CACHE": "1",
},
)
# Stream build logs
for chunk in build_logs:
if "stream" in chunk:
print(chunk["stream"].strip())
elif "error" in chunk:
raise RuntimeError(f"Build failed: {chunk['error']}")
print(f"Build complete: {image.id[:12]}")
# Get image details
image_info = client.images.get(full_image)
size_bytes = image_info.attrs["Size"]
digest = image_info.attrs.get("RepoDigests", ["unknown"])[0]
result = {
"image": full_image,
"digest": digest,
"size_bytes": size_bytes,
"image_id": image.id,
}
if push:
print(f"Pushing to {registry}...")
# Authenticate with registry based on hostname
_authenticate_registry(registry, client)
# Push image
push_logs = client.images.push(
repository=f"{registry}/{image_name}",
tag=tag,
stream=True,
decode=True,
)
for chunk in push_logs:
if "status" in chunk:
print(f" {chunk['status']}: {chunk.get('progress', '')}")
if "error" in chunk:
raise RuntimeError(f"Push failed: {chunk['error']}")
print(f"Push complete: {full_image}")
# Get digest after push
pushed_image = client.images.get(full_image)
result["digest"] = pushed_image.attrs.get("RepoDigests", ["unknown"])[0]
return result
# Prepare docker config for Kaniko if pushing to ECR
def _authenticate_registry(registry: str, client): docker_config_dir = None
"""Authenticate Docker client with the target registry.""" if push and ("ecr.aws" in registry or "ecr." in registry):
import subprocess print(f"Configuring ECR authentication...")
if "ecr.aws" in registry or "ecr." in registry:
# AWS ECR (public or private)
region = os.getenv("AWS_REGION", "us-east-1") region = os.getenv("AWS_REGION", "us-east-1")
print(f"Authenticating with AWS ECR ({region})...")
subprocess.run([
"aws", "ecr", "get-login-password", "--region", region
], check=True, capture_output=True)
# Note: Modal's aws-ecr-credentials secret should handle this via env vars
@app.function( ecr = boto3.client("ecr", region_name=region)
image=backend_image, auth = ecr.get_authorization_token()
timeout=120, token = auth["authorizationData"][0]["authorizationToken"]
secrets=[modal.Secret.from_name("aws-secrets")], username, password = base64.b64decode(token).decode().split(":")
)
def test_image(registry: str, image_name: str, tag: str) -> dict: config = {
"""Test the built image by running a quick health check.""" "auths": {
import docker registry: {
"username": username,
full_image = f"{registry}/{image_name}:{tag}" "password": password,
client = docker.from_env() }
}
print(f"Pulling {full_image} for testing...") }
client.images.pull(full_image)
docker_config_dir = PROJECT_ROOT / ".kaniko"
print("Running basic smoke test...") docker_config_dir.mkdir(exist_ok=True)
output = client.containers.run( (docker_config_dir / "config.json").write_text(json.dumps(config))
full_image, print(f"ECR auth configured for {registry}")
command=[
"python", "-c", # Build with Kaniko
"import sys; print('Python', sys.version); " kaniko_cmd = [
"import cv2; print('OpenCV', cv2.__version__); " "/usr/local/bin/kaniko",
], "--context", str(PROJECT_ROOT),
remove=True, "--dockerfile", str(DOCKERFILE_PATH),
detach=False, "--destination", full_image,
) "--platform", platform,
"--verbosity", "info",
]
if docker_config_dir:
kaniko_cmd.extend(["--docker-config", str(docker_config_dir)])
print(f"Running Kaniko: {' '.join(kaniko_cmd)}")
result = subprocess.run(kaniko_cmd, capture_output=True, text=True)
print(result.stdout)
if result.returncode != 0:
raise RuntimeError(f"Kaniko build failed:\n{result.stderr}")
print(f"Push complete: {full_image}")
return { return {
"status": "healthy",
"output": output.decode().strip(),
"image": full_image, "image": full_image,
"digest": "unknown",
"size_bytes": 0,
"image_id": "kaniko-build",
} }
@@ -237,18 +190,21 @@ def main(
test: bool = False, test: bool = False,
): ):
""" """
Build and push backend Docker image via Modal. Build and push backend Docker image via Kaniko on Modal.
Args: Args:
registry: Container registry (default: public.ecr.aws/vkist-project) registry: Container registry (default: public.ecr.aws/vkist-project)
image_name: Image name (default: vkist-backend) image_name: Image name (default: vkist-backend)
tag: Image tag (default: latest) tag: Image tag (default: latest)
push: Push to registry (default: true) push: Push to registry (default: true)
platform: Target platform (default: linux/amd64) platform: Target platform (default: linux/amd64)
test: Run health check after build (default: false) test: Run health check after build (default: false, unsupported with Kaniko)
""" """
if test:
print("WARNING: post-push testing is not supported with Kaniko. Skipping test.")
print("=" * 60) print("=" * 60)
print("MSK-Lumina Backend Docker Build on Modal") print("MSK-Lumina Backend Docker Build on Modal (Kaniko)")
print("=" * 60) print("=" * 60)
print(f"Registry: {registry}") print(f"Registry: {registry}")
print(f"Image: {image_name}") print(f"Image: {image_name}")
@@ -257,8 +213,7 @@ def main(
print(f"Platform: {platform}") print(f"Platform: {platform}")
print(f"Test: {test}") print(f"Test: {test}")
print("=" * 60) print("=" * 60)
# Build and push
result = build_and_push.remote( result = build_and_push.remote(
registry=registry, registry=registry,
image_name=image_name, image_name=image_name,
@@ -266,26 +221,18 @@ def main(
push=push, push=push,
platform=platform, platform=platform,
) )
print("\n" + "=" * 60) print("\n" + "=" * 60)
print("BUILD COMPLETE") print("BUILD COMPLETE")
print("=" * 60) print("=" * 60)
print(f"Image: {result['image']}") print(f"Image: {result['image']}")
print(f"Digest: {result['digest']}") print(f"Digest: {result['digest']}")
print(f"Size: {result['size_bytes'] / 1024 / 1024:.1f} MB")
print("=" * 60) print("=" * 60)
# Output for CI/CD parsing (GitHub Actions / Gitea Actions format) # Output for CI/CD parsing (GitHub Actions / Gitea Actions format)
print(f"::set-output name=image::{result['image']}") print(f"::set-output name=image::{result['image']}")
print(f"::set-output name=digest::{result['digest']}") print(f"::set-output name=digest::{result['digest']}")
print(f"::set-output name=size_mb::{result['size_bytes'] / 1024 / 1024:.1f}") print(f"::set-output name=size_mb::{result['size_bytes'] / 1024 / 1024:.1f}")
# Optional test
if test and push:
print("\nRunning health check...")
test_result = test_image.remote(registry, image_name, tag)
print(f"Health check: {test_result['status']}")
print(f"Output: {test_result['output']}")
if __name__ == "__main__": if __name__ == "__main__":
@@ -299,4 +246,4 @@ if __name__ == "__main__":
print(" modal deploy gitea_modal_build.py") print(" modal deploy gitea_modal_build.py")
else: else:
print("Use: modal run gitea_modal_build.py [options]") print("Use: modal run gitea_modal_build.py [options]")
print("Or: modal deploy gitea_modal_build.py") print("Or: modal deploy gitea_modal_build.py")