Harden deploy with flock and verify frontend-only incremental rebuilds.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
eric
parent
cc5b9ab1ad
commit
f6019e050f
@ -28,6 +28,7 @@ const LOGS = [
|
|||||||
"行程支持下载 Markdown 文件",
|
"行程支持下载 Markdown 文件",
|
||||||
"新增饮食饮水注意事项",
|
"新增饮食饮水注意事项",
|
||||||
"修复生产环境 CSS 解析失败导致的页面异常",
|
"修复生产环境 CSS 解析失败导致的页面异常",
|
||||||
|
"部署加速：BuildKit 缓存 + 按变更增量重建前后端",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@ -4,6 +4,7 @@ Fast incremental deploy for nomadro.
|
|||||||
- Pulls latest origin/dev1
|
- Pulls latest origin/dev1
|
||||||
- Rebuilds only services touched since last deploy
|
- Rebuilds only services touched since last deploy
|
||||||
- Uses BuildKit cache mounts (npm/pip)
|
- Uses BuildKit cache mounts (npm/pip)
|
||||||
|
- Single remote critical section under flock (no concurrent collide)
|
||||||
- Polls health instead of fixed long sleep
|
- Polls health instead of fixed long sleep
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@ -27,7 +28,7 @@ COMPOSE = "docker-compose.prod.yml"
|
|||||||
|
|
||||||
|
|
||||||
def run(client: paramiko.SSHClient, cmd: str, timeout: int = 900) -> tuple[int, str, str]:
|
def run(client: paramiko.SSHClient, cmd: str, timeout: int = 900) -> tuple[int, str, str]:
|
||||||
print("\n>>", cmd[:180])
|
print("\n>>", cmd[:200].replace("\n", " "))
|
||||||
_, stdout, stderr = client.exec_command(cmd, timeout=timeout)
|
_, stdout, stderr = client.exec_command(cmd, timeout=timeout)
|
||||||
out = stdout.read().decode("utf-8", errors="replace")
|
out = stdout.read().decode("utf-8", errors="replace")
|
||||||
err = stderr.read().decode("utf-8", errors="replace")
|
err = stderr.read().decode("utf-8", errors="replace")
|
||||||
@ -35,142 +36,141 @@ def run(client: paramiko.SSHClient, cmd: str, timeout: int = 900) -> tuple[int,
|
|||||||
if out.strip():
|
if out.strip():
|
||||||
print(out[-5000:])
|
print(out[-5000:])
|
||||||
if err.strip():
|
if err.strip():
|
||||||
# build logs often go to stderr
|
print("LOG:", err[-2000:])
|
||||||
print("LOG:", err[-2500:])
|
|
||||||
print("exit:", code)
|
print("exit:", code)
|
||||||
return code, out, err
|
return code, out, err
|
||||||
|
|
||||||
|
|
||||||
|
REMOTE_SCRIPT = r"""
|
||||||
|
set -euo pipefail
|
||||||
|
exec 9>/var/lock/nomadro-deploy.lock
|
||||||
|
if ! flock -n 9; then
|
||||||
|
echo "LOCK_BUSY"
|
||||||
|
exit 75
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOMAIN="__DOMAIN__"
|
||||||
|
GIT="__GIT__"
|
||||||
|
REMOTE="__REMOTE__"
|
||||||
|
MARKER="__MARKER__"
|
||||||
|
COMPOSE="__COMPOSE__"
|
||||||
|
|
||||||
|
export DOMAIN \
|
||||||
|
NEXT_PUBLIC_API_URL="https://${DOMAIN}/api/v1" \
|
||||||
|
NEXT_PUBLIC_SITE_URL="https://${DOMAIN}" \
|
||||||
|
CORS_ORIGINS="https://${DOMAIN},http://${DOMAIN}" \
|
||||||
|
DOCKER_BUILDKIT=1 \
|
||||||
|
COMPOSE_DOCKER_CLI_BUILD=1 \
|
||||||
|
BUILDKIT_PROGRESS=plain
|
||||||
|
|
||||||
|
cd "$REMOTE"
|
||||||
|
git remote set-url origin "$GIT"
|
||||||
|
git fetch --depth=80 origin
|
||||||
|
git checkout -B dev1 origin/dev1
|
||||||
|
git reset --hard origin/dev1
|
||||||
|
|
||||||
|
NEW=$(git rev-parse HEAD)
|
||||||
|
OLD=$(cat "$MARKER" 2>/dev/null || true)
|
||||||
|
|
||||||
|
MODE=full
|
||||||
|
if [ -n "$OLD" ] && git cat-file -e "${OLD}^{commit}" 2>/dev/null; then
|
||||||
|
CHANGED=$(git diff --name-only "$OLD" "$NEW" || true)
|
||||||
|
FE=0; BE=0
|
||||||
|
echo "$CHANGED" | grep -qE '^frontend/' && FE=1 || true
|
||||||
|
echo "$CHANGED" | grep -qE '^backend/' && BE=1 || true
|
||||||
|
echo "$CHANGED" | grep -qE '^docker-compose\.prod\.yml$' && FE=1 && BE=1 || true
|
||||||
|
if [ -z "$CHANGED" ]; then
|
||||||
|
MODE=noop
|
||||||
|
elif [ "$FE" = 1 ] && [ "$BE" = 0 ]; then
|
||||||
|
MODE=frontend
|
||||||
|
elif [ "$BE" = 1 ] && [ "$FE" = 0 ]; then
|
||||||
|
MODE=backend
|
||||||
|
elif [ "$FE" = 1 ] || [ "$BE" = 1 ]; then
|
||||||
|
MODE=full
|
||||||
|
else
|
||||||
|
MODE=noop
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "MODE=$MODE"
|
||||||
|
echo "NEW=$NEW"
|
||||||
|
echo "OLD=${OLD:-none}"
|
||||||
|
|
||||||
|
case "$MODE" in
|
||||||
|
noop)
|
||||||
|
echo "No app changes — skip build"
|
||||||
|
;;
|
||||||
|
frontend)
|
||||||
|
docker compose -f "$COMPOSE" build frontend
|
||||||
|
docker compose -f "$COMPOSE" up -d --no-deps frontend
|
||||||
|
;;
|
||||||
|
backend)
|
||||||
|
docker compose -f "$COMPOSE" build backend
|
||||||
|
docker compose -f "$COMPOSE" up -d --no-deps backend
|
||||||
|
;;
|
||||||
|
full)
|
||||||
|
docker compose -f "$COMPOSE" build
|
||||||
|
docker compose -f "$COMPOSE" up -d
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
ok=0
|
||||||
|
for i in $(seq 1 25); do
|
||||||
|
web=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3055/ || true)
|
||||||
|
api=$(curl -s http://127.0.0.1:8055/api/v1/health || true)
|
||||||
|
echo "try=$i web=$web api=$api"
|
||||||
|
if echo "$api" | grep -q '"status":"ok"' && [ "$web" = "200" ]; then
|
||||||
|
ok=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}' | grep nomadflow || true
|
||||||
|
curl -skI "https://${DOMAIN}" | head -6
|
||||||
|
curl -sk "https://${DOMAIN}/api/v1/health"
|
||||||
|
echo "$NEW" > "$MARKER"
|
||||||
|
|
||||||
|
[ "$ok" = 1 ]
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
t0 = time.time()
|
t0 = time.time()
|
||||||
client = paramiko.SSHClient()
|
client = paramiko.SSHClient()
|
||||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
client.connect(HOST, username=USER, password=PASS, timeout=30)
|
client.connect(HOST, username=USER, password=PASS, timeout=30)
|
||||||
|
|
||||||
env = (
|
script = (
|
||||||
f"DOMAIN={DOMAIN} "
|
REMOTE_SCRIPT
|
||||||
f"NEXT_PUBLIC_API_URL=https://{DOMAIN}/api/v1 "
|
.replace("__DOMAIN__", DOMAIN)
|
||||||
f"NEXT_PUBLIC_SITE_URL=https://{DOMAIN} "
|
.replace("__GIT__", GIT)
|
||||||
f"CORS_ORIGINS=https://{DOMAIN},http://{DOMAIN} "
|
.replace("__REMOTE__", REMOTE)
|
||||||
"DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 "
|
.replace("__MARKER__", MARKER)
|
||||||
"BUILDKIT_PROGRESS=plain"
|
.replace("__COMPOSE__", COMPOSE)
|
||||||
)
|
)
|
||||||
|
|
||||||
code, _, _ = run(
|
# Upload and run as one locked remote job
|
||||||
|
code, out, _ = run(
|
||||||
client,
|
client,
|
||||||
f"cd {REMOTE} && git remote set-url origin {GIT} && "
|
f"cat > /tmp/nomadro-deploy.sh <<'EOF'\n{script}\nEOF\nbash /tmp/nomadro-deploy.sh",
|
||||||
f"git fetch --depth=50 origin && "
|
timeout=900,
|
||||||
f"git checkout -B dev1 origin/dev1 && "
|
|
||||||
f"git reset --hard origin/dev1",
|
|
||||||
)
|
)
|
||||||
if code != 0:
|
|
||||||
client.close()
|
|
||||||
return code
|
|
||||||
|
|
||||||
# Decide what changed vs last successful deploy
|
mode = "unknown"
|
||||||
detect = f"""
|
|
||||||
set -e
|
|
||||||
cd {REMOTE}
|
|
||||||
NEW=$(git rev-parse HEAD)
|
|
||||||
OLD=$(cat {MARKER} 2>/dev/null || true)
|
|
||||||
if [ -z "$OLD" ] || ! git cat-file -e "$OLD^{{commit}}" 2>/dev/null; then
|
|
||||||
echo "MODE=full"
|
|
||||||
echo "NEW=$NEW"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
CHANGED=$(git diff --name-only "$OLD" "$NEW" || true)
|
|
||||||
echo "OLD=$OLD"
|
|
||||||
echo "NEW=$NEW"
|
|
||||||
echo "CHANGED_BEGIN"
|
|
||||||
echo "$CHANGED"
|
|
||||||
echo "CHANGED_END"
|
|
||||||
FE=0; BE=0
|
|
||||||
echo "$CHANGED" | grep -qE '^frontend/' && FE=1 || true
|
|
||||||
echo "$CHANGED" | grep -qE '^backend/' && BE=1 || true
|
|
||||||
echo "$CHANGED" | grep -qE '^docker-compose\\.prod\\.yml$' && FE=1 && BE=1 || true
|
|
||||||
if [ -z "$CHANGED" ]; then
|
|
||||||
echo "MODE=noop"
|
|
||||||
elif [ "$FE" = 1 ] && [ "$BE" = 0 ]; then
|
|
||||||
echo "MODE=frontend"
|
|
||||||
elif [ "$BE" = 1 ] && [ "$FE" = 0 ]; then
|
|
||||||
echo "MODE=backend"
|
|
||||||
elif [ "$FE" = 1 ] || [ "$BE" = 1 ]; then
|
|
||||||
echo "MODE=full"
|
|
||||||
else
|
|
||||||
echo "MODE=noop"
|
|
||||||
fi
|
|
||||||
"""
|
|
||||||
code, out, _ = run(client, detect, timeout=60)
|
|
||||||
if code != 0:
|
|
||||||
client.close()
|
|
||||||
return code
|
|
||||||
|
|
||||||
mode = "full"
|
|
||||||
new_sha = ""
|
|
||||||
for line in out.splitlines():
|
for line in out.splitlines():
|
||||||
if line.startswith("MODE="):
|
if line.startswith("MODE="):
|
||||||
mode = line.split("=", 1)[1].strip()
|
mode = line.split("=", 1)[1].strip()
|
||||||
if line.startswith("NEW="):
|
|
||||||
new_sha = line.split("=", 1)[1].strip()
|
|
||||||
|
|
||||||
print(f"\nDeploy mode: {mode}")
|
|
||||||
|
|
||||||
if mode == "noop":
|
|
||||||
print("No changes since last deploy — skip build")
|
|
||||||
elif mode == "frontend":
|
|
||||||
code, _, _ = run(
|
|
||||||
client,
|
|
||||||
f"cd {REMOTE} && export {env} && "
|
|
||||||
f"docker compose -f {COMPOSE} build frontend && "
|
|
||||||
f"docker compose -f {COMPOSE} up -d --no-deps frontend",
|
|
||||||
timeout=900,
|
|
||||||
)
|
|
||||||
elif mode == "backend":
|
|
||||||
code, _, _ = run(
|
|
||||||
client,
|
|
||||||
f"cd {REMOTE} && export {env} && "
|
|
||||||
f"docker compose -f {COMPOSE} build backend && "
|
|
||||||
f"docker compose -f {COMPOSE} up -d --no-deps backend",
|
|
||||||
timeout=600,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
code, _, _ = run(
|
|
||||||
client,
|
|
||||||
f"cd {REMOTE} && export {env} && "
|
|
||||||
f"docker compose -f {COMPOSE} build && "
|
|
||||||
f"docker compose -f {COMPOSE} up -d",
|
|
||||||
timeout=900,
|
|
||||||
)
|
|
||||||
|
|
||||||
if code != 0:
|
|
||||||
client.close()
|
|
||||||
return code
|
|
||||||
|
|
||||||
# Fast health poll (max ~20s) instead of fixed sleep 12
|
|
||||||
health = f"""
|
|
||||||
ok=0
|
|
||||||
for i in $(seq 1 20); do
|
|
||||||
web=$(curl -s -o /dev/null -w '%{{http_code}}' http://127.0.0.1:3055/ || true)
|
|
||||||
api=$(curl -s http://127.0.0.1:8055/api/v1/health || true)
|
|
||||||
echo "try=$i web=$web api=$api"
|
|
||||||
echo "$api" | grep -q '"status":"ok"' && [ "$web" = "200" ] && ok=1 && break
|
|
||||||
sleep 1
|
|
||||||
done
|
|
||||||
docker ps --format 'table {{{{.Names}}}}\\t{{{{.Status}}}}\\t{{{{.Ports}}}}' | grep nomadflow || true
|
|
||||||
curl -skI https://{DOMAIN} | head -6
|
|
||||||
curl -sk https://{DOMAIN}/api/v1/health
|
|
||||||
[ "$ok" = 1 ]
|
|
||||||
"""
|
|
||||||
code, _, _ = run(client, health, timeout=60)
|
|
||||||
if code != 0:
|
|
||||||
client.close()
|
|
||||||
return code
|
|
||||||
|
|
||||||
if new_sha:
|
|
||||||
run(client, f"echo {new_sha} > {MARKER}", timeout=20)
|
|
||||||
|
|
||||||
client.close()
|
client.close()
|
||||||
elapsed = int(time.time() - t0)
|
elapsed = int(time.time() - t0)
|
||||||
|
if code == 75:
|
||||||
|
print("Deploy aborted: another deploy is in progress")
|
||||||
|
return 75
|
||||||
|
if code != 0:
|
||||||
|
print(f"\nDeploy FAILED in {elapsed}s (mode={mode})")
|
||||||
|
return code
|
||||||
|
|
||||||
print(f"\nDeploy OK in {elapsed}s (mode={mode})")
|
print(f"\nDeploy OK in {elapsed}s (mode={mode})")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user