From 8af0675895aa4f8948636c381b9753df11aa891f Mon Sep 17 00:00:00 2001 From: Abdulmuizz Hamzat <103611443+buildwithhamzat@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:05:10 +0100 Subject: [PATCH] fix: minio bucket seed race (#820) * fix(workflows): retry MinIO bucket creation to survive post-boot format window A freshly duplicated cio-minio's HTTP listener accepts connections before it finishes formatting its storage backend (confirmed via its own boot logs: "Formatting 1st pool, 1 set(s), 1 drives per set."), so a bucket-creation call landing in that window can transiently fail with a misleading "specified bucket does not exist" response even though auth and connectivity (mc alias set) already succeeded. Retry bucket creation up to 10 times with a 5s backoff instead of failing the whole run on what's usually just a not-ready-yet response. * fix(workflows): truncate env name slug to fit Railway's length limit railway environment new failed with "Error in name - Invalid input" for a 48-char name (pr-cursor-notes-workspace-feedback-6d09-0e6aa7c9) derived from a long auto-generated branch name. Railway docs cite a 32-char limit for service names; environment names hit the same wall. The collision-resistant hash suffix (added for the earlier CodeRabbit finding) is what actually guarantees uniqueness -- the slug is only there for readability -- so cap the slug at 18 chars instead of dropping the hash. pr-<=18>- stays at or under 30 chars. * fix(workflows): retry mc anonymous set for the same boot-race window The mb retry loop worked as intended in the latest test run (attempt 1 failed on all three buckets during cio-minio's post-boot format window, attempt 2 succeeded), but the very next call -- anonymous set download on cio/media -- hit the identical "specified bucket does not exist" error right after mb had just succeeded. Same underlying race, one command later. Wrap it in the same retry pattern. * update workflow * fix: silen error on db:seed * update workflow * update workflow * update workflow --- .../workflows/railway-preview-teardown.yml | 2 +- .github/workflows/railway-preview.yml | 144 +++++++++++++++--- 2 files changed, 122 insertions(+), 24 deletions(-) diff --git a/.github/workflows/railway-preview-teardown.yml b/.github/workflows/railway-preview-teardown.yml index 92a6b690..ba827948 100644 --- a/.github/workflows/railway-preview-teardown.yml +++ b/.github/workflows/railway-preview-teardown.yml @@ -45,7 +45,7 @@ jobs: env: HEAD_REF: ${{ github.event.pull_request.head.ref }} run: | - slug=$(echo "$HEAD_REF" | tr '[:upper:]/_.' '[:lower:]---' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g; s/^-//; s/-$//') + slug=$(echo "$HEAD_REF" | tr '[:upper:]/_.' '[:lower:]---' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g; s/^-//; s/-$//' | cut -c1-18 | sed 's/-$//') hash=$(echo -n "$HEAD_REF" | sha256sum | cut -c1-8) env_name="pr-${slug}-${hash}" echo "Attempting to delete environment: ${env_name}" diff --git a/.github/workflows/railway-preview.yml b/.github/workflows/railway-preview.yml index 600918f4..403c2faa 100644 --- a/.github/workflows/railway-preview.yml +++ b/.github/workflows/railway-preview.yml @@ -71,11 +71,14 @@ jobs: # BRANCH is bound via env (not interpolated into the script) so an attacker-supplied # value can't be executed as shell. "feat/foo_bar" -> "pr-feat-foo-bar-" — the # slug alone is lossy (e.g. "foo/bar" and "foo_bar" both slug to "foo-bar"), so a - # hash of the original branch string disambiguates collisions. + # hash of the original branch string disambiguates collisions. Slug is capped at 18 + # chars so "pr--" stays under Railway's ~32-char environment name limit + # even for long auto-generated branch names — the hash (not the slug) is what + # actually guarantees uniqueness, so truncating the slug is safe. env: BRANCH: ${{ inputs.branch }} run: | - slug=$(echo "$BRANCH" | tr '[:upper:]/_.' '[:lower:]---' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g; s/^-//; s/-$//') + slug=$(echo "$BRANCH" | tr '[:upper:]/_.' '[:lower:]---' | sed 's/[^a-z0-9-]//g' | sed 's/--*/-/g; s/^-//; s/-$//' | cut -c1-18 | sed 's/-$//') hash=$(echo -n "$BRANCH" | sha256sum | cut -c1-8) echo "name=pr-${slug}-${hash}" >> "$GITHUB_OUTPUT" @@ -118,10 +121,26 @@ jobs: - name: Link project + preview environment if: ${{ inputs.action == 'deploy' }} + # A brand-new environment (first run for a branch) can take a moment to become + # visible to `railway link` right after `railway environment new` returns — same + # duplication-propagation lag as the cio-minio ServiceInstance race below. Retry + # instead of failing the run. env: ENV_NAME: ${{ steps.env.outputs.name }} run: | - railway link --project "$RAILWAY_PROJECT_ID" --workspace "$RAILWAY_WORKSPACE" --environment "$ENV_NAME" + for attempt in $(seq 1 10); do + if output=$(railway link --project "$RAILWAY_PROJECT_ID" --workspace "$RAILWAY_WORKSPACE" --environment "$ENV_NAME" 2>&1); then + echo "$output" + break + fi + echo "$output" + if [ "$attempt" -eq 10 ]; then + echo "::error::Environment $ENV_NAME never became linkable after 10 attempts." + exit 1 + fi + echo "Environment not ready yet (attempt $attempt/10) — retrying in 5s..." + sleep 5 + done - name: Pin the Dockerfile per service if: ${{ inputs.action == 'deploy' }} @@ -140,18 +159,43 @@ jobs: # but the public domain doesn't, so generate one here (same as "Report the preview # URL" does for cio-dashboard). `--ignore-existing` keeps re-deploys safe. run: | - # --port 9000: the minio/minio image exposes both 9000 (S3 API) and 9001 - # (console); pin it so domain generation doesn't land on the wrong one. - railway domain --service cio-minio --port 9000 || true - # `cut -d= -f2-` (not `awk -F= '{print $2}'`) so a value containing '=' — common in - # base64/auto-generated secrets like MINIO_ROOT_PASSWORD — isn't silently truncated. - MINIO_HOST=$(railway variables --service cio-minio --kv | grep '^RAILWAY_PUBLIC_DOMAIN=' | cut -d= -f2-) + # A freshly duplicated environment's cio-minio ServiceInstance (volume-backed, like + # Postgres) can take a few seconds to materialize after `railway environment new + # --duplicate` returns, so `railway domain`/`variables` can fail with "ServiceInstance + # not found" immediately after duplication. Retry instead of failing the run. + MINIO_HOST="" + for attempt in $(seq 1 10); do + # --port 9000: the minio/minio image exposes both 9000 (S3 API) and 9001 + # (console); pin it so domain generation doesn't land on the wrong one. + if output=$(railway domain --service cio-minio --port 9000 2>&1); then + echo "$output" + else + echo "$output" + if ! echo "$output" | grep -qi "already exists"; then + if [ "$attempt" -eq 10 ]; then + echo "::error::cio-minio ServiceInstance never became available after 10 attempts." + exit 1 + fi + echo "cio-minio not ready yet (attempt $attempt/10) — retrying in 5s..." + sleep 5 + continue + fi + fi + # `cut -d= -f2-` (not `awk -F= '{print $2}'`) so a value containing '=' — common in + # base64/auto-generated secrets like MINIO_ROOT_PASSWORD — isn't silently truncated. + MINIO_HOST=$(railway variables --service cio-minio --kv | grep '^RAILWAY_PUBLIC_DOMAIN=' | cut -d= -f2-) + if [ -n "$MINIO_HOST" ]; then + break + fi + if [ "$attempt" -eq 10 ]; then + echo "::error::cio-minio has no public domain after 10 attempts — 'railway domain' may have failed." + exit 1 + fi + echo "cio-minio has no public domain yet (attempt $attempt/10) — retrying in 5s..." + sleep 5 + done MINIO_USER=$(railway variables --service cio-minio --kv | grep '^MINIO_ROOT_USER=' | cut -d= -f2-) MINIO_PASS=$(railway variables --service cio-minio --kv | grep '^MINIO_ROOT_PASSWORD=' | cut -d= -f2-) - if [ -z "$MINIO_HOST" ]; then - echo "::error::cio-minio has no public domain — 'railway domain' may have failed." - exit 1 - fi # Verify mc against MinIO's published SHA256 before executing it. curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc -o mc curl -sSL https://dl.min.io/client/mc/release/linux-amd64/mc.sha256sum -o mc.sha256sum @@ -162,8 +206,39 @@ jobs: # route wildcard bucket subdomains, so virtual-host style (mc's "auto" default) # fails bucket creation with a misleading "does not exist" error. ./mc alias set cio "https://${MINIO_HOST}" "$MINIO_USER" "$MINIO_PASS" --path on - ./mc mb cio/videos cio/documents cio/media --ignore-existing - ./mc anonymous set download cio/media + # A freshly duplicated cio-minio's HTTP listener accepts connections before it + # finishes formatting its storage backend ("Formatting 1st pool..." in its boot + # logs), so bucket creation can transiently fail right after boot. Retry instead + # of failing the run on what's usually just a not-ready-yet response. + for attempt in $(seq 1 10); do + if output=$(./mc mb cio/videos cio/documents cio/media --ignore-existing 2>&1); then + echo "$output" + break + fi + echo "$output" + if [ "$attempt" -eq 10 ]; then + echo "::error::cio-minio bucket creation failed after 10 attempts." + exit 1 + fi + echo "Bucket creation failed (attempt $attempt/10) — cio-minio may still be initializing; retrying in 5s..." + sleep 5 + done + # Same boot-race as above can hit the very next call too, even right after mb + # just succeeded — cio-minio's storage layer can take a moment past that point + # to fully settle. + for attempt in $(seq 1 10); do + if output=$(./mc anonymous set download cio/media 2>&1); then + echo "$output" + break + fi + echo "$output" + if [ "$attempt" -eq 10 ]; then + echo "::error::cio-minio anonymous-download policy failed after 10 attempts." + exit 1 + fi + echo "Anonymous policy set failed (attempt $attempt/10) — retrying in 5s..." + sleep 5 + done - name: Deploy each app service from this branch if: ${{ inputs.action == 'deploy' }} @@ -176,17 +251,40 @@ jobs: echo "::endgroup::" done + - name: Setup pnpm + if: ${{ inputs.action == 'deploy' && inputs.seed }} + uses: pnpm/action-setup@v4 + + - name: Setup Node.js + if: ${{ inputs.action == 'deploy' && inputs.seed }} + uses: actions/setup-node@v4 + with: + node-version: '20' + cache: 'pnpm' + + - name: Install dependencies + if: ${{ inputs.action == 'deploy' && inputs.seed }} + run: pnpm install --frozen-lockfile + + - name: Build workspace dependencies + if: ${{ inputs.action == 'deploy' && inputs.seed }} + # @cio/db imports compiled output (e.g. @cio/utils/dist/...) from its workspace + # deps, not source — they must be built before the seed script can import them. + run: pnpm --filter @cio/db^... build + - name: Seed the full demo dataset (optional) if: ${{ inputs.action == 'deploy' && inputs.seed }} - # SPIKE: `railway run` injects the service's env vars and runs the command ON THIS - # RUNNER, so it needs a DB URL reachable from GitHub (Railway's public TCP proxy, - # DATABASE_PUBLIC_URL) — NOT the private railway.internal one. If cio-api only - # exposes a private DATABASE_URL, add DATABASE_PUBLIC_URL to this step, or drop - # this step and rely on the api boot-time db:setup (migrations + essential seed). + # cio-api only carries the private DATABASE_URL; `railway run` executes on this + # GitHub runner, so seeding needs Postgres's own DATABASE_PUBLIC_URL instead + # (railway.internal isn't reachable from here). run: | - corepack enable - railway run --service cio-api -- pnpm --filter @cio/db db:setup:seed || \ - echo "Demo seed failed (likely DB not reachable from runner) — migrations + essential seed still ran on api boot." + PUBLIC_DB_URL=$(railway variables --service Postgres --kv | grep '^DATABASE_PUBLIC_URL=' | cut -d= -f2-) + if [ -z "$PUBLIC_DB_URL" ]; then + echo "::warning::Postgres has no DATABASE_PUBLIC_URL in this environment — skipping full seed; only essential seed ran on api boot." + else + railway run --service cio-api -- env DATABASE_URL="$PUBLIC_DB_URL" pnpm --filter @cio/db db:setup:seed || \ + echo "Demo seed failed (likely DB not reachable from runner) — migrations + essential seed still ran on api boot." + fi - name: Report the preview URL if: ${{ inputs.action == 'deploy' }}