Harden deploy with flock and verify frontend-only incremental rebuilds.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
eric
parent
cc5b9ab1ad
commit
f6019e050f
@ -28,6 +28,7 @@ const LOGS = [
|
||||
"行程支持下载 Markdown 文件",
|
||||
"新增饮食饮水注意事项",
|
||||
"修复生产环境 CSS 解析失败导致的页面异常",
|
||||
"部署加速：BuildKit 缓存 + 按变更增量重建前后端",
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
@ -4,6 +4,7 @@ Fast incremental deploy for nomadro.
|
||||
- Pulls latest origin/dev1
|
||||
- Rebuilds only services touched since last deploy
|
||||
- Uses BuildKit cache mounts (npm/pip)
|
||||
- Single remote critical section under flock (no concurrent collide)
|
||||
- Polls health instead of fixed long sleep
|
||||
"""
|
||||
|
||||
@ -27,7 +28,7 @@ COMPOSE = "docker-compose.prod.yml"
|
||||
|
||||
|
||||
def run(client: paramiko.SSHClient, cmd: str, timeout: int = 900) -> tuple[int, str, str]:
|
||||
print("\n>>", cmd[:180])
|
||||
print("\n>>", cmd[:200].replace("\n", " "))
|
||||
_, stdout, stderr = client.exec_command(cmd, timeout=timeout)
|
||||
out = stdout.read().decode("utf-8", errors="replace")
|
||||
err = stderr.read().decode("utf-8", errors="replace")
|
||||
@ -35,142 +36,141 @@ def run(client: paramiko.SSHClient, cmd: str, timeout: int = 900) -> tuple[int,
|
||||
if out.strip():
|
||||
print(out[-5000:])
|
||||
if err.strip():
|
||||
# build logs often go to stderr
|
||||
print("LOG:", err[-2500:])
|
||||
print("LOG:", err[-2000:])
|
||||
print("exit:", code)
|
||||
return code, out, err
|
||||
|
||||
|
||||
REMOTE_SCRIPT = r"""
|
||||
set -euo pipefail
|
||||
exec 9>/var/lock/nomadro-deploy.lock
|
||||
if ! flock -n 9; then
|
||||
echo "LOCK_BUSY"
|
||||
exit 75
|
||||
fi
|
||||
|
||||
DOMAIN="__DOMAIN__"
|
||||
GIT="__GIT__"
|
||||
REMOTE="__REMOTE__"
|
||||
MARKER="__MARKER__"
|
||||
COMPOSE="__COMPOSE__"
|
||||
|
||||
export DOMAIN \
|
||||
NEXT_PUBLIC_API_URL="https://${DOMAIN}/api/v1" \
|
||||
NEXT_PUBLIC_SITE_URL="https://${DOMAIN}" \
|
||||
CORS_ORIGINS="https://${DOMAIN},http://${DOMAIN}" \
|
||||
DOCKER_BUILDKIT=1 \
|
||||
COMPOSE_DOCKER_CLI_BUILD=1 \
|
||||
BUILDKIT_PROGRESS=plain
|
||||
|
||||
cd "$REMOTE"
|
||||
git remote set-url origin "$GIT"
|
||||
git fetch --depth=80 origin
|
||||
git checkout -B dev1 origin/dev1
|
||||
git reset --hard origin/dev1
|
||||
|
||||
NEW=$(git rev-parse HEAD)
|
||||
OLD=$(cat "$MARKER" 2>/dev/null || true)
|
||||
|
||||
MODE=full
|
||||
if [ -n "$OLD" ] && git cat-file -e "${OLD}^{commit}" 2>/dev/null; then
|
||||
CHANGED=$(git diff --name-only "$OLD" "$NEW" || true)
|
||||
FE=0; BE=0
|
||||
echo "$CHANGED" | grep -qE '^frontend/' && FE=1 || true
|
||||
echo "$CHANGED" | grep -qE '^backend/' && BE=1 || true
|
||||
echo "$CHANGED" | grep -qE '^docker-compose\.prod\.yml$' && FE=1 && BE=1 || true
|
||||
if [ -z "$CHANGED" ]; then
|
||||
MODE=noop
|
||||
elif [ "$FE" = 1 ] && [ "$BE" = 0 ]; then
|
||||
MODE=frontend
|
||||
elif [ "$BE" = 1 ] && [ "$FE" = 0 ]; then
|
||||
MODE=backend
|
||||
elif [ "$FE" = 1 ] || [ "$BE" = 1 ]; then
|
||||
MODE=full
|
||||
else
|
||||
MODE=noop
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "MODE=$MODE"
|
||||
echo "NEW=$NEW"
|
||||
echo "OLD=${OLD:-none}"
|
||||
|
||||
case "$MODE" in
|
||||
noop)
|
||||
echo "No app changes — skip build"
|
||||
;;
|
||||
frontend)
|
||||
docker compose -f "$COMPOSE" build frontend
|
||||
docker compose -f "$COMPOSE" up -d --no-deps frontend
|
||||
;;
|
||||
backend)
|
||||
docker compose -f "$COMPOSE" build backend
|
||||
docker compose -f "$COMPOSE" up -d --no-deps backend
|
||||
;;
|
||||
full)
|
||||
docker compose -f "$COMPOSE" build
|
||||
docker compose -f "$COMPOSE" up -d
|
||||
;;
|
||||
esac
|
||||
|
||||
ok=0
|
||||
for i in $(seq 1 25); do
|
||||
web=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3055/ || true)
|
||||
api=$(curl -s http://127.0.0.1:8055/api/v1/health || true)
|
||||
echo "try=$i web=$web api=$api"
|
||||
if echo "$api" | grep -q '"status":"ok"' && [ "$web" = "200" ]; then
|
||||
ok=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | grep nomadflow || true
|
||||
curl -skI "https://${DOMAIN}" | head -6
|
||||
curl -sk "https://${DOMAIN}/api/v1/health"
|
||||
echo "$NEW" > "$MARKER"
|
||||
|
||||
[ "$ok" = 1 ]
|
||||
"""
|
||||
|
||||
|
||||
def main() -> int:
|
||||
t0 = time.time()
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
client.connect(HOST, username=USER, password=PASS, timeout=30)
|
||||
|
||||
env = (
|
||||
f"DOMAIN={DOMAIN} "
|
||||
f"NEXT_PUBLIC_API_URL=https://{DOMAIN}/api/v1 "
|
||||
f"NEXT_PUBLIC_SITE_URL=https://{DOMAIN} "
|
||||
f"CORS_ORIGINS=https://{DOMAIN},http://{DOMAIN} "
|
||||
"DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 "
|
||||
"BUILDKIT_PROGRESS=plain"
|
||||
script = (
|
||||
REMOTE_SCRIPT
|
||||
.replace("__DOMAIN__", DOMAIN)
|
||||
.replace("__GIT__", GIT)
|
||||
.replace("__REMOTE__", REMOTE)
|
||||
.replace("__MARKER__", MARKER)
|
||||
.replace("__COMPOSE__", COMPOSE)
|
||||
)
|
||||
|
||||
code, _, _ = run(
|
||||
# Upload and run as one locked remote job
|
||||
code, out, _ = run(
|
||||
client,
|
||||
f"cd {REMOTE} && git remote set-url origin {GIT} && "
|
||||
f"git fetch --depth=50 origin && "
|
||||
f"git checkout -B dev1 origin/dev1 && "
|
||||
f"git reset --hard origin/dev1",
|
||||
f"cat > /tmp/nomadro-deploy.sh <<'EOF'\n{script}\nEOF\nbash /tmp/nomadro-deploy.sh",
|
||||
timeout=900,
|
||||
)
|
||||
if code != 0:
|
||||
client.close()
|
||||
return code
|
||||
|
||||
# Decide what changed vs last successful deploy
|
||||
detect = f"""
|
||||
set -e
|
||||
cd {REMOTE}
|
||||
NEW=$(git rev-parse HEAD)
|
||||
OLD=$(cat {MARKER} 2>/dev/null || true)
|
||||
if [ -z "$OLD" ] || ! git cat-file -e "$OLD^{{commit}}" 2>/dev/null; then
|
||||
echo "MODE=full"
|
||||
echo "NEW=$NEW"
|
||||
exit 0
|
||||
fi
|
||||
CHANGED=$(git diff --name-only "$OLD" "$NEW" || true)
|
||||
echo "OLD=$OLD"
|
||||
echo "NEW=$NEW"
|
||||
echo "CHANGED_BEGIN"
|
||||
echo "$CHANGED"
|
||||
echo "CHANGED_END"
|
||||
FE=0; BE=0
|
||||
echo "$CHANGED" | grep -qE '^frontend/' && FE=1 || true
|
||||
echo "$CHANGED" | grep -qE '^backend/' && BE=1 || true
|
||||
echo "$CHANGED" | grep -qE '^docker-compose\\.prod\\.yml$' && FE=1 && BE=1 || true
|
||||
if [ -z "$CHANGED" ]; then
|
||||
echo "MODE=noop"
|
||||
elif [ "$FE" = 1 ] && [ "$BE" = 0 ]; then
|
||||
echo "MODE=frontend"
|
||||
elif [ "$BE" = 1 ] && [ "$FE" = 0 ]; then
|
||||
echo "MODE=backend"
|
||||
elif [ "$FE" = 1 ] || [ "$BE" = 1 ]; then
|
||||
echo "MODE=full"
|
||||
else
|
||||
echo "MODE=noop"
|
||||
fi
|
||||
"""
|
||||
code, out, _ = run(client, detect, timeout=60)
|
||||
if code != 0:
|
||||
client.close()
|
||||
return code
|
||||
|
||||
mode = "full"
|
||||
new_sha = ""
|
||||
mode = "unknown"
|
||||
for line in out.splitlines():
|
||||
if line.startswith("MODE="):
|
||||
mode = line.split("=", 1)[1].strip()
|
||||
if line.startswith("NEW="):
|
||||
new_sha = line.split("=", 1)[1].strip()
|
||||
|
||||
print(f"\nDeploy mode: {mode}")
|
||||
|
||||
if mode == "noop":
|
||||
print("No changes since last deploy — skip build")
|
||||
elif mode == "frontend":
|
||||
code, _, _ = run(
|
||||
client,
|
||||
f"cd {REMOTE} && export {env} && "
|
||||
f"docker compose -f {COMPOSE} build frontend && "
|
||||
f"docker compose -f {COMPOSE} up -d --no-deps frontend",
|
||||
timeout=900,
|
||||
)
|
||||
elif mode == "backend":
|
||||
code, _, _ = run(
|
||||
client,
|
||||
f"cd {REMOTE} && export {env} && "
|
||||
f"docker compose -f {COMPOSE} build backend && "
|
||||
f"docker compose -f {COMPOSE} up -d --no-deps backend",
|
||||
timeout=600,
|
||||
)
|
||||
else:
|
||||
code, _, _ = run(
|
||||
client,
|
||||
f"cd {REMOTE} && export {env} && "
|
||||
f"docker compose -f {COMPOSE} build && "
|
||||
f"docker compose -f {COMPOSE} up -d",
|
||||
timeout=900,
|
||||
)
|
||||
|
||||
if code != 0:
|
||||
client.close()
|
||||
return code
|
||||
|
||||
# Fast health poll (max ~20s) instead of fixed sleep 12
|
||||
health = f"""
|
||||
ok=0
|
||||
for i in $(seq 1 20); do
|
||||
web=$(curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:3055/ || true)
|
||||
api=$(curl -s http://127.0.0.1:8055/api/v1/health || true)
|
||||
echo "try=$i web=$web api=$api"
|
||||
echo "$api" | grep -q '"status":"ok"' && [ "$web" = "200" ] && ok=1 && break
|
||||
sleep 1
|
||||
done
|
||||
docker ps --format 'table {{{{.Names}}}}\\t{{{{.Status}}}}\\t{{{{.Ports}}}}' | grep nomadflow || true
|
||||
curl -skI https://{DOMAIN} | head -6
|
||||
curl -sk https://{DOMAIN}/api/v1/health
|
||||
[ "$ok" = 1 ]
|
||||
"""
|
||||
code, _, _ = run(client, health, timeout=60)
|
||||
if code != 0:
|
||||
client.close()
|
||||
return code
|
||||
|
||||
if new_sha:
|
||||
run(client, f"echo {new_sha} > {MARKER}", timeout=20)
|
||||
|
||||
client.close()
|
||||
elapsed = int(time.time() - t0)
|
||||
if code == 75:
|
||||
print("Deploy aborted: another deploy is in progress")
|
||||
return 75
|
||||
if code != 0:
|
||||
print(f"\nDeploy FAILED in {elapsed}s (mode={mode})")
|
||||
return code
|
||||
|
||||
print(f"\nDeploy OK in {elapsed}s (mode={mode})")
|
||||
return 0
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user