From ddba364bd2b6a6e90e7b001d5a75d91c8414052f Mon Sep 17 00:00:00 2001 From: Rotimi Best Date: Sat, 15 Aug 2026 23:31:41 +0100 Subject: [PATCH] perf(ci): parallelize Railway preview deploys (#909) --- .github/scripts/railway-preview-deploy.sh | 139 ++++++++++++++++ .../scripts/test-railway-preview-deploy.sh | 156 ++++++++++++++++++ .github/workflows/railway-preview.yml | 54 +----- 3 files changed, 300 insertions(+), 49 deletions(-) create mode 100755 .github/scripts/railway-preview-deploy.sh create mode 100755 .github/scripts/test-railway-preview-deploy.sh diff --git a/.github/scripts/railway-preview-deploy.sh b/.github/scripts/railway-preview-deploy.sh new file mode 100755 index 00000000..7980c4b9 --- /dev/null +++ b/.github/scripts/railway-preview-deploy.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Deploy Railway preview app services concurrently and recover from the known +# `railway up --ci` log-streaming false negative (railwayapp/cli#696). +# +# Required env: +# APP_SERVICES — space-separated service names (e.g. "cio-api cio-dashboard cio-jobs") +# RUNNER_TEMP — writable directory for per-service logs +# +# Optional env (mainly for tests): +# RAILWAY_DEPLOY_MAX_ATTEMPTS — poll attempts (default: 60) +# RAILWAY_DEPLOY_SLEEP_SECONDS — sleep between polls (default: 10) + +set -euo pipefail + +RAILWAY_DEPLOY_MAX_ATTEMPTS="${RAILWAY_DEPLOY_MAX_ATTEMPTS:-60}" +RAILWAY_DEPLOY_SLEEP_SECONDS="${RAILWAY_DEPLOY_SLEEP_SECONDS:-10}" + +# Returns 0 when the deployment reached SUCCESS, 1 otherwise. +poll_deployment_status() { + local service="$1" + local deploy_id="$2" + local status="" + local attempt + + for attempt in $(seq 1 "$RAILWAY_DEPLOY_MAX_ATTEMPTS"); do + status=$( + railway deployment list --service "$service" --json 2>/dev/null | + jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty' + ) + + case "$status" in + SUCCESS) + echo "Deployment $deploy_id for $service actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing." + return 0 + ;; + FAILED|CRASHED|REMOVED) + echo "::error::$service deployment $deploy_id did not succeed (status: $status)." + return 1 + ;; + esac + + if [ "$attempt" -eq "$RAILWAY_DEPLOY_MAX_ATTEMPTS" ]; then + echo "::error::$service deployment $deploy_id never reached a terminal status after $RAILWAY_DEPLOY_MAX_ATTEMPTS attempts (last seen: ${status:-unknown})." + return 1 + fi + + echo "Deployment $deploy_id for $service not finished yet (status: ${status:-unknown}, attempt $attempt/$RAILWAY_DEPLOY_MAX_ATTEMPTS) — retrying in ${RAILWAY_DEPLOY_SLEEP_SECONDS}s..." + sleep "$RAILWAY_DEPLOY_SLEEP_SECONDS" + done + + return 1 +} + +# Resolve a non-zero `railway up` exit using the captured log. +# Only the known log-streaming glitch is eligible for status polling. +# Returns 0 on recovered SUCCESS, 1 on failure. +resolve_railway_up_failure() { + local service="$1" + local deployment_log="$2" + local deploy_id + + if ! grep -q "Failed to stream build logs:" "$deployment_log"; then + echo "::error::railway up failed for $service for a reason other than the known log-streaming glitch — not polling stale deployment status." + return 1 + fi + + deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$deployment_log" | head -n1) + if [ -z "$deploy_id" ]; then + echo "::error::railway up failed for $service and no deployment id could be recovered from its output — cannot verify actual status." + return 1 + fi + + echo "railway up reported failure for $service due to the known log-streaming glitch; checking status of deployment $deploy_id..." + poll_deployment_status "$service" "$deploy_id" +} + +deploy_app_services_in_parallel() { + local -a services=() + local -a deployment_pids=() + local -a deployment_logs=() + local service + local deployment_log + local service_index + local railway_status + local deployment_failed=false + + if [ -z "${APP_SERVICES:-}" ]; then + echo "::error::APP_SERVICES is required." + return 1 + fi + + if [ -z "${RUNNER_TEMP:-}" ]; then + echo "::error::RUNNER_TEMP is required." + return 1 + fi + + # shellcheck disable=SC2206 + services=($APP_SERVICES) + + for service in "${services[@]}"; do + deployment_log="$RUNNER_TEMP/railway-${service}.log" + echo "Starting deployment for $service..." + railway up --service "$service" --ci >"$deployment_log" 2>&1 & + deployment_pids+=("$!") + deployment_logs+=("$deployment_log") + done + + for service_index in "${!services[@]}"; do + service="${services[$service_index]}" + deployment_log="${deployment_logs[$service_index]}" + + if wait "${deployment_pids[$service_index]}"; then + railway_status=0 + else + railway_status=1 + fi + + echo "::group::railway up $service" + cat "$deployment_log" + + if [ "$railway_status" -ne 0 ]; then + if ! resolve_railway_up_failure "$service" "$deployment_log"; then + deployment_failed=true + fi + fi + + echo "::endgroup::" + done + + if [ "$deployment_failed" = true ]; then + echo "::error::One or more Railway app deployments failed." + return 1 + fi +} + +# Only auto-run when executed directly (not when sourced by tests). +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + deploy_app_services_in_parallel +fi diff --git a/.github/scripts/test-railway-preview-deploy.sh b/.github/scripts/test-railway-preview-deploy.sh new file mode 100755 index 00000000..2d3ab8ea --- /dev/null +++ b/.github/scripts/test-railway-preview-deploy.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Mocked coverage for concurrent Railway preview deploys and log-stream recovery. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=.github/scripts/railway-preview-deploy.sh +source "$SCRIPT_DIR/railway-preview-deploy.sh" + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +MOCK_BIN="$TMP_ROOT/bin" +mkdir -p "$MOCK_BIN" +export PATH="$MOCK_BIN:$PATH" + +export RUNNER_TEMP="$TMP_ROOT/runner-temp" +mkdir -p "$RUNNER_TEMP" +export RAILWAY_DEPLOY_MAX_ATTEMPTS=3 +export RAILWAY_DEPLOY_SLEEP_SECONDS=0 + +write_mock_railway() { + local mode="$1" + + cat >"$MOCK_BIN/railway" <&2 + exit 99 + ;; + esac + ;; + deployment) + service="" + while [ \$# -gt 0 ]; do + case "\$1" in + --service) + service="\$2" + shift 2 + ;; + *) + shift + ;; + esac + done + + case "\$mode:\$service" in + stream-success:cio-api) + printf '%s\\n' '[{"id":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","status":"SUCCESS"}]' + ;; + stream-failed:cio-api) + printf '%s\\n' '[{"id":"dddddddd-dddd-dddd-dddd-dddddddddddd","status":"FAILED"}]' + ;; + *) + printf '%s\\n' '[]' + ;; + esac + ;; + *) + echo "unexpected railway command: \$cmd" >&2 + exit 99 + ;; +esac +EOF + + chmod +x "$MOCK_BIN/railway" +} + +assert_exit() { + local expected="$1" + local label="$2" + shift 2 + + set +e + "$@" + local actual=$? + set -e + + if [ "$actual" -ne "$expected" ]; then + echo "FAIL: $label (expected exit $expected, got $actual)" >&2 + exit 1 + fi + + echo "PASS: $label" +} + +export APP_SERVICES="cio-api cio-dashboard cio-jobs" + +write_mock_railway success +assert_exit 0 "concurrent success" deploy_app_services_in_parallel + +write_mock_railway stream-success +assert_exit 0 "stream error followed by SUCCESS" deploy_app_services_in_parallel + +write_mock_railway stream-failed +assert_exit 1 "stream error followed by FAILED" deploy_app_services_in_parallel + +write_mock_railway other-failure +assert_exit 1 "non-stream failure does not poll" deploy_app_services_in_parallel + +write_mock_railway partial-failure +assert_exit 1 "partial concurrent failure" deploy_app_services_in_parallel + +echo "All railway preview deploy recovery mocks passed." diff --git a/.github/workflows/railway-preview.yml b/.github/workflows/railway-preview.yml index 6fedc551..2a75dbda 100644 --- a/.github/workflows/railway-preview.yml +++ b/.github/workflows/railway-preview.yml @@ -246,62 +246,18 @@ jobs: sleep 5 done - - name: Deploy each app service from this branch + - name: Deploy app services from this branch in parallel if: ${{ inputs.action == 'deploy' }} - # `--ci` (not `--detach`) blocks until each service's deploy/healthcheck succeeds, - # so the URL printed later is actually live. Trade-off: builds run sequentially. + # `--ci` (not `--detach`) keeps each process attached until its deployment + # succeeds or fails. Run the independent app builds concurrently, then wait for + # every service so the URL printed later is actually live. # # `railway up --ci` can exit 1 on a log-streaming glitch (railwayapp/cli#696) even # though the deploy succeeded. Only on that specific error do we poll the real # deployment status instead of trusting the exit code — any other failure means no # new deployment exists yet, so it fails immediately rather than risking a match # against a stale, unrelated deployment. - run: | - for svc in $APP_SERVICES; do - echo "::group::railway up $svc" - log_file=$(mktemp) - railway up --service "$svc" --ci 2>&1 | tee "$log_file" - railway_status=${PIPESTATUS[0]} - if [ "$railway_status" -ne 0 ]; then - if grep -q "Failed to stream build logs:" "$log_file"; then - deploy_id=$(grep -oP '(?<=[?&]id=)[0-9a-fA-F-]+' "$log_file" | head -n1) - if [ -z "$deploy_id" ]; then - echo "::error::railway up failed for $svc and no deployment id could be recovered from its output — cannot verify actual status." - rm -f "$log_file" - exit 1 - fi - echo "railway up reported failure for $svc due to the known log-streaming glitch; checking status of deployment $deploy_id..." - status="" - for attempt in $(seq 1 60); do - status=$(railway deployment list --service "$svc" --json 2>/dev/null | jq -r --arg id "$deploy_id" '.[] | select(.id == $id) | .status // empty') - case "$status" in - SUCCESS) - echo "Deployment $deploy_id for $svc actually succeeded (status: SUCCESS) — the CLI error was just a log-streaming glitch. Continuing." - break - ;; - FAILED|CRASHED|REMOVED) - echo "::error::$svc deployment $deploy_id did not succeed (status: $status)." - rm -f "$log_file" - exit 1 - ;; - esac - if [ "$attempt" -eq 60 ]; then - echo "::error::$svc deployment $deploy_id never reached a terminal status after 60 attempts (last seen: ${status:-unknown})." - rm -f "$log_file" - exit 1 - fi - echo "Deployment $deploy_id for $svc not finished yet (status: ${status:-unknown}, attempt $attempt/60) — retrying in 10s..." - sleep 10 - done - else - echo "::error::railway up failed for $svc for a reason other than the known log-streaming glitch — not polling stale deployment status." - rm -f "$log_file" - exit 1 - fi - fi - rm -f "$log_file" - echo "::endgroup::" - done + run: bash .github/scripts/railway-preview-deploy.sh - name: Setup pnpm if: ${{ inputs.action == 'deploy' && inputs.seed }}