perf(ci): parallelize Railway preview deploys (#909)
This commit is contained in:
Rotimi Best
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy Railway preview app services concurrently and recover from the known
|
||||
# `railway up --ci` log-streaming false negative (railwayapp/cli#696).
|
||||
#
|
||||
# Required env:
|
||||
# APP_SERVICES — space-separated service names (e.g. "cio-api cio-dashboard cio-jobs")
|
||||
# RUNNER_TEMP — writable directory for per-service logs
|
||||
#
|
||||
# Optional env (mainly for tests):
|
||||
# RAILWAY_DEPLOY_MAX_ATTEMPTS — poll attempts (default: 60)
|
||||
# RAILWAY_DEPLOY_SLEEP_SECONDS — sleep between polls (default: 10)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
RAILWAY_DEPLOY_MAX_ATTEMPTS="${RAILWAY_DEPLOY_MAX_ATTEMPTS:-60}"
|
||||
RAILWAY_DEPLOY_SLEEP_SECONDS="${RAILWAY_DEPLOY_SLEEP_SECONDS:-10}"
|
||||
|
||||
# Returns 0 when the deployment reached SUCCESS, 1 otherwise.
|
||||
poll_deployment_status() {
|
||||
local service="$1"
|
||||
local deploy_id="$2"
|
||||
local status=""
|
||||
local attempt
|
||||
|
||||
for attempt in $(seq 1 "$RAILWAY_DEPLOY_MAX_ATTEMPTS"); do
|
||||
status=$(
|
||||
railway deployment list --service "$service" --json 2>/dev/null |
|
||||
jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty'
|
||||
)
|
||||
|
||||
case "$status" in
|
||||
SUCCESS)
|
||||
echo "Deployment $deploy_id for $service actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing."
|
||||
return 0
|
||||
;;
|
||||
FAILED|CRASHED|REMOVED)
|
||||
echo "::error::$service deployment $deploy_id did not succeed (status: $status)."
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "$attempt" -eq "$RAILWAY_DEPLOY_MAX_ATTEMPTS" ]; then
|
||||
echo "::error::$service deployment $deploy_id never reached a terminal status after $RAILWAY_DEPLOY_MAX_ATTEMPTS attempts (last seen: ${status:-unknown})."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Deployment $deploy_id for $service not finished yet (status: ${status:-unknown}, attempt $attempt/$RAILWAY_DEPLOY_MAX_ATTEMPTS) — retrying in ${RAILWAY_DEPLOY_SLEEP_SECONDS}s..."
|
||||
sleep "$RAILWAY_DEPLOY_SLEEP_SECONDS"
|
||||
done
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Resolve a non-zero `railway up` exit using the captured log.
|
||||
# Only the known log-streaming glitch is eligible for status polling.
|
||||
# Returns 0 on recovered SUCCESS, 1 on failure.
|
||||
resolve_railway_up_failure() {
|
||||
local service="$1"
|
||||
local deployment_log="$2"
|
||||
local deploy_id
|
||||
|
||||
if ! grep -q "Failed to stream build logs:" "$deployment_log"; then
|
||||
echo "::error::railway up failed for $service for a reason other than the known log-streaming glitch — not polling stale deployment status."
|
||||
return 1
|
||||
fi
|
||||
|
||||
deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$deployment_log" | head -n1)
|
||||
if [ -z "$deploy_id" ]; then
|
||||
echo "::error::railway up failed for $service and no deployment id could be recovered from its output — cannot verify actual status."
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "railway up reported failure for $service due to the known log-streaming glitch; checking status of deployment $deploy_id..."
|
||||
poll_deployment_status "$service" "$deploy_id"
|
||||
}
|
||||
|
||||
deploy_app_services_in_parallel() {
|
||||
local -a services=()
|
||||
local -a deployment_pids=()
|
||||
local -a deployment_logs=()
|
||||
local service
|
||||
local deployment_log
|
||||
local service_index
|
||||
local railway_status
|
||||
local deployment_failed=false
|
||||
|
||||
if [ -z "${APP_SERVICES:-}" ]; then
|
||||
echo "::error::APP_SERVICES is required."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ -z "${RUNNER_TEMP:-}" ]; then
|
||||
echo "::error::RUNNER_TEMP is required."
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2206
|
||||
services=($APP_SERVICES)
|
||||
|
||||
for service in "${services[@]}"; do
|
||||
deployment_log="$RUNNER_TEMP/railway-${service}.log"
|
||||
echo "Starting deployment for $service..."
|
||||
railway up --service "$service" --ci >"$deployment_log" 2>&1 &
|
||||
deployment_pids+=("$!")
|
||||
deployment_logs+=("$deployment_log")
|
||||
done
|
||||
|
||||
for service_index in "${!services[@]}"; do
|
||||
service="${services[$service_index]}"
|
||||
deployment_log="${deployment_logs[$service_index]}"
|
||||
|
||||
if wait "${deployment_pids[$service_index]}"; then
|
||||
railway_status=0
|
||||
else
|
||||
railway_status=1
|
||||
fi
|
||||
|
||||
echo "::group::railway up $service"
|
||||
cat "$deployment_log"
|
||||
|
||||
if [ "$railway_status" -ne 0 ]; then
|
||||
if ! resolve_railway_up_failure "$service" "$deployment_log"; then
|
||||
deployment_failed=true
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "::endgroup::"
|
||||
done
|
||||
|
||||
if [ "$deployment_failed" = true ]; then
|
||||
echo "::error::One or more Railway app deployments failed."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Only auto-run when executed directly (not when sourced by tests).
|
||||
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
||||
deploy_app_services_in_parallel
|
||||
fi
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
# Mocked coverage for concurrent Railway preview deploys and log-stream recovery.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=.github/scripts/railway-preview-deploy.sh
|
||||
source "$SCRIPT_DIR/railway-preview-deploy.sh"
|
||||
|
||||
TMP_ROOT="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_ROOT"' EXIT
|
||||
|
||||
MOCK_BIN="$TMP_ROOT/bin"
|
||||
mkdir -p "$MOCK_BIN"
|
||||
export PATH="$MOCK_BIN:$PATH"
|
||||
|
||||
export RUNNER_TEMP="$TMP_ROOT/runner-temp"
|
||||
mkdir -p "$RUNNER_TEMP"
|
||||
export RAILWAY_DEPLOY_MAX_ATTEMPTS=3
|
||||
export RAILWAY_DEPLOY_SLEEP_SECONDS=0
|
||||
|
||||
write_mock_railway() {
|
||||
local mode="$1"
|
||||
|
||||
cat >"$MOCK_BIN/railway" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
mode="$mode"
|
||||
cmd="\${1:-}"
|
||||
|
||||
case "\$cmd" in
|
||||
up)
|
||||
service=""
|
||||
while [ \$# -gt 0 ]; do
|
||||
case "\$1" in
|
||||
--service)
|
||||
service="\$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "\$mode:\$service" in
|
||||
success:*)
|
||||
echo "Deploying \$service..."
|
||||
echo "https://railway.com/project/p/service/s/deployments?id=11111111-1111-1111-1111-111111111111"
|
||||
exit 0
|
||||
;;
|
||||
stream-success:cio-api)
|
||||
echo "Deploying cio-api..."
|
||||
echo "https://railway.com/project/p/service/s/deployments?id=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
|
||||
echo "Failed to stream build logs: connection reset"
|
||||
exit 1
|
||||
;;
|
||||
stream-failed:cio-api)
|
||||
echo "Deploying cio-api..."
|
||||
echo "https://railway.com/project/p/service/s/deployments?id=dddddddd-dddd-dddd-dddd-dddddddddddd"
|
||||
echo "Failed to stream build logs: connection reset"
|
||||
exit 1
|
||||
;;
|
||||
other-failure:cio-api)
|
||||
echo "Deploying cio-api..."
|
||||
echo "error: build failed before a deployment id was published"
|
||||
exit 1
|
||||
;;
|
||||
partial-failure:cio-dashboard)
|
||||
echo "Deploying cio-dashboard..."
|
||||
echo "error: dashboard build failed"
|
||||
exit 1
|
||||
;;
|
||||
stream-success:*|stream-failed:*|other-failure:*|partial-failure:*)
|
||||
echo "Deploying \$service..."
|
||||
echo "https://railway.com/project/p/service/s/deployments?id=eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unexpected railway up mode/service: \$mode/\$service" >&2
|
||||
exit 99
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
deployment)
|
||||
service=""
|
||||
while [ \$# -gt 0 ]; do
|
||||
case "\$1" in
|
||||
--service)
|
||||
service="\$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "\$mode:\$service" in
|
||||
stream-success:cio-api)
|
||||
printf '%s\\n' '[{"id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","status":"SUCCESS"}]'
|
||||
;;
|
||||
stream-failed:cio-api)
|
||||
printf '%s\\n' '[{"id":"dddddddd-dddd-dddd-dddd-dddddddddddd","status":"FAILED"}]'
|
||||
;;
|
||||
*)
|
||||
printf '%s\\n' '[]'
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
*)
|
||||
echo "unexpected railway command: \$cmd" >&2
|
||||
exit 99
|
||||
;;
|
||||
esac
|
||||
EOF
|
||||
|
||||
chmod +x "$MOCK_BIN/railway"
|
||||
}
|
||||
|
||||
assert_exit() {
|
||||
local expected="$1"
|
||||
local label="$2"
|
||||
shift 2
|
||||
|
||||
set +e
|
||||
"$@"
|
||||
local actual=$?
|
||||
set -e
|
||||
|
||||
if [ "$actual" -ne "$expected" ]; then
|
||||
echo "FAIL: $label (expected exit $expected, got $actual)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: $label"
|
||||
}
|
||||
|
||||
export APP_SERVICES="cio-api cio-dashboard cio-jobs"
|
||||
|
||||
write_mock_railway success
|
||||
assert_exit 0 "concurrent success" deploy_app_services_in_parallel
|
||||
|
||||
write_mock_railway stream-success
|
||||
assert_exit 0 "stream error followed by SUCCESS" deploy_app_services_in_parallel
|
||||
|
||||
write_mock_railway stream-failed
|
||||
assert_exit 1 "stream error followed by FAILED" deploy_app_services_in_parallel
|
||||
|
||||
write_mock_railway other-failure
|
||||
assert_exit 1 "non-stream failure does not poll" deploy_app_services_in_parallel
|
||||
|
||||
write_mock_railway partial-failure
|
||||
assert_exit 1 "partial concurrent failure" deploy_app_services_in_parallel
|
||||
|
||||
echo "All railway preview deploy recovery mocks passed."
|
||||
Reference in New Issue
Block a user