fix(ci): serialize app pipelines and keep plan and apply consistent

.github/workflows/expand-terraform.yml

@@ -31,9 +31,12 @@ jobs:
     name: Generate Terraform
     runs-on: ubuntu-latest
     steps:
+      # Same-repo PRs: the branch itself, so the generated commit lands on it.
+      # Fork PRs: github.head_ref names a branch that exists only in the fork,
+      # so use the PR merge ref (github.ref) instead.
       - uses: actions/checkout@v6
         with:
-          ref: ${{ github.head_ref || github.ref }}
+          ref: ${{ (github.event.pull_request.head.repo.full_name == github.repository && github.head_ref) || github.ref }}
 
       # javaBin/platform is public: the caller's own GITHUB_TOKEN can read it,
       # so app repos need no GitHub App credentials for any pipeline step.
@@ -58,7 +61,8 @@ jobs:
 
       # Pushed with the caller's GITHUB_TOKEN (needs `contents: write` in the
       # app's deploy.yml). Pushes made with GITHUB_TOKEN never start another
-      # workflow run, so this cannot loop.
+      # workflow run, so this cannot loop. A rejected push (branch protection,
+      # read-only token, branch moved) fails with the fix in the log.
       - name: Commit and push generated files
         env:
           BRANCH: ${{ github.head_ref || github.ref_name }}

.github/workflows/javabin.yml

@@ -70,6 +70,16 @@ permissions:
   contents: write
   pull-requests: write
 
+# One pipeline per repo and ref. On the default branch runs queue instead of
+# racing: two quick merges would otherwise push generated Terraform over each
+# other, fight over the state lock or apply a stale saved plan. A newer run
+# replaces a queued one (it contains the same commits). PR runs cancel the
+# superseded run. The group name must stay distinct from any group in the
+# caller's deploy.yml, or the two deadlock.
+concurrency:
+  group: javabin-pipeline-${{ github.repository }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   # --------------------------------------------------------------------------
   # 1. Detect repo contents
@@ -104,6 +114,8 @@ jobs:
 
   # --------------------------------------------------------------------------
   # 3. Docker build + ECR push (default branch only, after builds pass)
+  #    Repos with app.yaml or terraform/ need a successful plan: a broken
+  #    app.yaml fails expand, and then nothing is built or deployed.
   # --------------------------------------------------------------------------
   docker-build:
     needs: [detect, build-jvm, build-ts, tf-plan, tf-apply]
@@ -114,7 +126,7 @@ jobs:
       needs.detect.outputs.has_docker == 'true' &&
       (needs.build-jvm.result == 'success' || needs.build-jvm.result == 'skipped') &&
       (needs.build-ts.result == 'success' || needs.build-ts.result == 'skipped') &&
-      needs.tf-plan.result != 'failure' &&
+      (needs.tf-plan.result == 'success' || (needs.detect.outputs.has_tf != 'true' && needs.detect.outputs.has_yaml != 'true')) &&
       (needs.tf-apply.result == 'success' || needs.tf-apply.result == 'skipped')
     uses: javaBin/platform/.github/workflows/docker-build.yml@main
     with:
@@ -156,7 +168,7 @@ jobs:
       plan_key: ${{ needs.tf-plan.outputs.plan_key }}
       plan_sha256: ${{ needs.tf-plan.outputs.plan_sha256 }}
       planned_sha: ${{ needs.tf-plan.outputs.planned_sha }}
-      team: ${{ needs.detect.outputs.app_team }}
+      team: ${{ needs.tf-plan.outputs.team || needs.detect.outputs.app_team }}
 
   # --------------------------------------------------------------------------
   # 7. ECS deploy (default branch only, after docker build + TF apply)

.github/workflows/platform-ci.yml

@@ -231,6 +231,12 @@ jobs:
         working-directory: ${{ env.TF_ROOT }}
         run: sh "${{ github.workspace }}/scripts/verify-plan.sh" tfplan "${{ needs.plan.outputs.plan_sha256 }}"
 
+      # Install the provider versions the plan was made with (the lock file
+      # is inside the verified plan; it is gitignored in the repo).
+      - name: Restore provider lock file from plan
+        working-directory: ${{ env.TF_ROOT }}
+        run: unzip -o -q tfplan .terraform.lock.hcl
+
       - name: Terraform Init
         working-directory: ${{ env.TF_ROOT }}
         run: terraform init -input=false

.github/workflows/tf-apply.yml

@@ -28,7 +28,7 @@ on:
         type: string
         default: "terraform"
       team:
-        description: "Team from app.yaml (the gate verifies the repo belongs to it)"
+        description: "Team tf-plan used (tf-plan's team output; the gate verifies the repo belongs to it)"
         type: string
         default: ""
 
@@ -40,6 +40,10 @@ jobs:
   apply:
     name: Terraform Apply
     runs-on: ubuntu-latest
+    # One apply per repo state at a time, also for callers outside javabin.yml.
+    concurrency:
+      group: javabin-tf-apply-${{ github.repository }}-${{ inputs.tf_root }}
+      cancel-in-progress: false
     env:
       PLAN_BUCKET: javabin-ci-plan-artifacts-${{ inputs.aws_account_id }}
     steps:
@@ -86,13 +90,30 @@ jobs:
         run: sh .platform/scripts/invoke-apply-gate.sh "${{ inputs.plan_key }}" "${{ github.event.repository.name }}" "${{ inputs.team }}"
 
       # Step 4: Apply with credentials from the gate Lambda
+      # Same backend.tf/providers.tf as tf-plan generated for repos that bring
+      # their own Terraform without them.
+      - name: Ensure Terraform boilerplate
+        env:
+          AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
+          AWS_REGION: ${{ inputs.aws_region || 'eu-central-1' }}
+          TEAM: ${{ inputs.team }}
+        run: sh .platform/scripts/ensure-tf-boilerplate.sh "${{ inputs.tf_root }}"
+
+      # The saved plan carries the dependency lock file tf-plan's init wrote,
+      # and it is covered by the SHA256 check above. Restoring it makes init
+      # install exactly the provider versions the plan was made with, even if
+      # a newer provider was released in between.
+      - name: Restore provider lock file from plan
+        working-directory: ${{ inputs.tf_root }}
+        run: unzip -o -q tfplan .terraform.lock.hcl
+
       - name: Terraform Init
         working-directory: ${{ inputs.tf_root }}
         run: terraform init -input=false
 
       - name: Terraform Apply
         working-directory: ${{ inputs.tf_root }}
-        run: terraform apply -auto-approve tfplan
+        run: terraform apply -auto-approve -lock-timeout=5m tfplan
 
       # Non-blocking: checks Cognito sign-in right after apply (sync errors,
       # redirect_mismatch, logout URL mismatch, unknown auth.groups). Waits

.github/workflows/tf-plan.yml

@@ -35,6 +35,9 @@ on:
       planned_sha:
         description: "Commit the plan was made from (includes generated Terraform)"
         value: ${{ jobs.plan.outputs.planned_sha }}
+      team:
+        description: "Team the CI broker verified for the repo (tf-apply uses the same state key)"
+        value: ${{ jobs.plan.outputs.team }}
 
 permissions:
   id-token: write
@@ -51,6 +54,7 @@ jobs:
       plan_sha256: ${{ steps.upload.outputs.plan_sha256 }}
       risk_level: ${{ steps.review.outputs.risk_level }}
       planned_sha: ${{ steps.sha.outputs.sha }}
+      team: ${{ steps.broker.outputs.team }}
     env:
       PLAN_BUCKET: javabin-ci-plan-artifacts-${{ inputs.aws_account_id }}
     steps:
@@ -59,10 +63,11 @@ jobs:
         run: echo "SESSION_NAME=$(echo "${GITHUB_ACTOR}-${GITHUB_SHA:0:8}-${GITHUB_RUN_ID}" | tr -c 'A-Za-z0-9+=,.@_\n-' '-' | head -c 64)" >> "$GITHUB_ENV"
 
       # Branch head, not github.sha: expand may have just committed generated
-      # Terraform on top of the triggering commit.
+      # Terraform on top of the triggering commit. Fork PRs use the PR merge
+      # ref: their github.head_ref exists only in the fork.
       - uses: actions/checkout@v6
         with:
-          ref: ${{ github.head_ref || github.ref }}
+          ref: ${{ (github.event.pull_request.head.repo.full_name == github.repository && github.head_ref) || github.ref }}
 
       - name: Record planned commit
         id: sha
@@ -84,7 +89,14 @@ jobs:
           sparse-checkout: scripts
           persist-credentials: false
 
+      # Fork and Dependabot PRs get no OIDC token: skip the AWS steps with a
+      # notice and only validate.
+      - name: Check AWS access
+        id: aws
+        run: sh .platform/scripts/check-aws-access.sh
+
       - name: Assume broker role via OIDC
+        if: steps.aws.outputs.available == 'true'
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: arn:aws:iam::${{ inputs.aws_account_id }}:role/javabin-ci-app-broker
@@ -93,21 +105,23 @@ jobs:
 
       - name: Get team credentials from broker
         id: broker
+        if: steps.aws.outputs.available == 'true'
         env:
           PROJECT: javabin
           APP_TEAM: ${{ inputs.team }}
         run: sh .platform/scripts/invoke-ci-broker.sh plan
 
       - name: Ensure Terraform boilerplate
+        if: steps.aws.outputs.available == 'true'
         env:
           AWS_ACCOUNT_ID: ${{ inputs.aws_account_id }}
           AWS_REGION: ${{ inputs.aws_region || 'eu-central-1' }}
-          GH_TOKEN: ${{ github.token }}
+          TEAM: ${{ steps.broker.outputs.team }}
         run: sh .platform/scripts/ensure-tf-boilerplate.sh "${{ inputs.tf_root }}"
 
       - name: Terraform Init
         working-directory: ${{ inputs.tf_root }}
-        run: terraform init -input=false
+        run: terraform init -input=false ${{ steps.aws.outputs.available != 'true' && '-backend=false' || '' }}
 
       - name: Terraform Validate
         working-directory: ${{ inputs.tf_root }}
@@ -117,9 +131,11 @@ jobs:
         working-directory: ${{ inputs.tf_root }}
         run: terraform fmt -check -recursive
 
+      # Waits for a lock held by a running apply instead of failing at once.
       - name: Terraform Plan
         id: plan
-        run: .platform/scripts/run-plan.sh "${{ inputs.tf_root }}"
+        if: steps.aws.outputs.available == 'true'
+        run: .platform/scripts/run-plan.sh "${{ inputs.tf_root }}" -lock-timeout=5m
 
       - name: Upload plan to S3
         id: upload
@@ -148,7 +164,7 @@ jobs:
         run: sh .platform/scripts/invoke-plan-review.sh "${{ steps.upload.outputs.plan_key }}"
 
       - name: Post plan + review to PR
-        if: github.event_name == 'pull_request'
+        if: github.event_name == 'pull_request' && steps.aws.outputs.available == 'true'
         env:
           GH_TOKEN: ${{ github.token }}
           PR_NUMBER: ${{ github.event.pull_request.number }}

CLAUDE.md

@@ -200,6 +200,9 @@ terraform/state/
 | `scripts/invoke-apply-gate.sh` | CI: invoke gate Lambda for apply credentials |
 | `scripts/run-plan.sh` | CI: terraform plan with exit code handling |
 | `scripts/upload-plan.sh` | CI: upload plan artifact to S3 |
+| `scripts/commit-generated-tf.sh` | CI (expand): commit + push generated Terraform; a rejected push fails with the reason and fix (tests: `scripts/tests/`) |
+| `scripts/ensure-tf-boilerplate.sh` | CI (app tf-plan + tf-apply): backend.tf/providers.tf for repos that bring their own Terraform, from the broker-verified `TEAM` (tests: `scripts/tests/`) |
+| `scripts/check-aws-access.sh` | CI (app tf-plan): skips AWS steps with a notice when the run has no OIDC token (fork/Dependabot PRs) (tests: `scripts/tests/`) |
 | `scripts/auth-doctor.py` | CI (after app apply) / local: checks an app's Cognito sign-in end to end from auth-status + HTTP probes (tests: `scripts/tests/`) |
 | `scripts/reconcile-app-auth.sh` | CI (after platform apply): queue an app-auth-sync reconcile |
 

docs/ci-workflow.md

@@ -30,6 +30,7 @@ Monday 08:00   ──► securityhub-summary Lambda ──► #javabin-infra-ale
 - Only on `push` to `main` with changes
 - Checks risk level: blocks on HIGH/FAILED/UNKNOWN
 - Downloads plan from S3, verifies SHA256 integrity
+- Restores `.terraform.lock.hcl` from the plan file (Terraform stores it there; it is gitignored in the repo), so `terraform init` installs the provider versions the plan was made with
 - `terraform apply -auto-approve tfplan`
 - Uses `environment: production` (GitHub environment protection)
 - OIDC → `javabin-ci-infra`. The role's trust also pins `job_workflow_ref` to `platform-ci.yml@refs/heads/main`, because the environment-based `sub` claim carries no branch. A workflow on another branch that declares `environment: production` cannot assume it.

docs/reusable-workflows.md

@@ -46,6 +46,30 @@ detect ──┬─ expand ──── tf-plan ── tf-apply ──┐
 - **PRs:** detect → build → expand (commits generated TF to the PR branch) → tf-plan (plan + LLM review) → PR comment
 - **Default branch** (push or manual run): detect → build → expand → tf-plan → tf-apply → docker-build → ecs-deploy
 
+`docker-build` (and so `ecs-deploy`) in a repo with `app.yaml` or `terraform/` runs only after a
+successful `tf-plan`, and after `tf-apply` when the plan had changes. A broken `app.yaml` fails
+`expand` and nothing is built or deployed, instead of pushing an image to an ECR repository that the
+skipped apply never created.
+
+### Concurrency
+
+`javabin.yml` runs one pipeline per repo and ref (group `javabin-pipeline-<repo>-<ref>`). On the
+default branch a second run waits for the first one, so two quick merges cannot push generated
+Terraform over each other, fight over the state lock or apply a stale saved plan. GitHub keeps at most
+one waiting run per group: a third push replaces the waiting run, and the newest run contains all its
+commits. A newer push to a pull request cancels the PR's running pipeline. Don't use a group named
+`javabin-pipeline-...` in your own `deploy.yml`: caller and called workflow would deadlock.
+`tf-apply.yml` also holds a per-repo group of its own for callers outside `javabin.yml`, and plan and
+apply wait up to 5 minutes for the state lock.
+
+### Pull requests from forks and Dependabot
+
+These runs get a read-only `GITHUB_TOKEN` and usually no OIDC token. `expand` checks out the PR
+merge ref, validates `app.yaml` and skips the push; `tf-plan` skips every AWS step with a notice
+("Terraform plan skipped") and only runs `terraform init -backend=false`, `validate` and `fmt -check`.
+The plan runs after merge. A Dependabot PR that does get `id-token: write` from `deploy.yml` plans
+like any other PR.
+
 Deploy jobs (`tf-apply`, `docker-build`, `ecs-deploy`, `eb-deploy`) run only when
 `github.ref` is `refs/heads/<default branch>` (`github.event.repository.default_branch`) and the
 event is not a pull request. The full ref is compared, so a tag that happens to be named like the
@@ -117,6 +141,21 @@ pnpm install + `build` + `test`. `build` and `test` are skipped when `package.js
 
 Runs `scripts/generate-modules.py`: validates `app.yaml` (unknown or unsupported fields fail with a message), writes `terraform/` and commits it back to the branch with `scripts/commit-generated-tf.sh`. The push uses the caller's `GITHUB_TOKEN`, so it never triggers another run. The commit message (`chore(terraform): update generated Terraform from app.yaml`) deliberately has no skip-CI marker: squash merges copy commit messages, and the marker would skip the merged PR's deploy. On PRs from forks the push is skipped.
 
+The push needs `contents: write` and a branch that accepts pushes from GitHub Actions. If it is
+rejected, `expand` fails and says why:
+
+- **Protected default branch** (branch protection or a ruleset): change `app.yaml` through pull
+  requests. `expand` commits the generated files to the PR branch, so the default branch is up to
+  date after merge and needs no push. If it is already out of date (a platform generator change since
+  the last run), open a PR with any change from the default branch and merge it.
+- **Branch moved** while the run generated: the run for the newer commit regenerates and pushes.
+- **Read-only token**: grant `contents: write` in `deploy.yml`.
+
+GitHub starts no workflow for commits pushed with `GITHUB_TOKEN`, so the generated commit on a PR has
+no checks of its own (the run that pushed it goes on to plan it). If the repo requires status checks,
+pull the branch and push another commit (`git commit --allow-empty`); that run finds nothing to
+regenerate and reports its checks on the new head.
+
 ### docker-build.yml
 
 Docker BuildKit build + push to ECR repo `{team}-{app_name}`. Uses GitHub Actions cache.
@@ -136,11 +175,13 @@ Docker BuildKit build + push to ECR repo `{team}-{app_name}`. Uses GitHub Action
 Plans the committed `terraform/` at the branch head (after expand), uploads the plan to S3 with its SHA256, has the `javabin-plan-review` Lambda review it (Bedrock Claude Haiku 5.5, platform credentials; it writes `risk.json`) and posts plan + review to the PR. A review that cannot run after retries is `FAILED`: re-run the workflow, or request an override if it persists.
 
 **AWS role:** `javabin-ci-app-broker` (OIDC) → broker Lambda → `javabin-ci-plan-{team}` (read-only: refresh, state read, state lock, plan upload; tf-plan runs on any branch, so it cannot change resources); the review step uses the broker OIDC role to invoke `javabin-plan-review`
-**Outputs:** `has_changes`, `plan_key`, `plan_sha256`, `risk_level`, `planned_sha`
+Repos that bring their own `terraform/` without `backend.tf`/`providers.tf` get them from `scripts/ensure-tf-boilerplate.sh`, with state key `apps/<team>/<repo>` and the team the broker verified (not a GitHub API lookup with the run's token). tf-apply generates the same files with the same team.
+
+**Outputs:** `has_changes`, `plan_key`, `plan_sha256`, `risk_level`, `planned_sha`, `team`
 
 ### tf-apply.yml
 
-Checks out the commit tf-plan planned, verifies the plan SHA256 and asks the apply-gate Lambda for credentials (HIGH/FAILED risk needs a board-member override), then applies.
+Checks out the commit tf-plan planned, verifies the plan SHA256 and asks the apply-gate Lambda for credentials (HIGH/FAILED risk needs a board-member override), then applies. Before `terraform init` it restores `.terraform.lock.hcl` from the saved plan (Terraform stores the lock file in the plan, and the SHA256 check covers it), so apply installs exactly the provider versions the plan used even though app repos gitignore the lock file and pin the AWS provider only to `~> 5.0`.
 
 **AWS role:** `javabin-ci-apply-gate` (OIDC) → apply-gate Lambda → `javabin-ci-team-{team}`
 

scripts/check-aws-access.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Decide whether this job can reach AWS via GitHub OIDC.
+#
+# Usage: check-aws-access.sh
+#
+# GitHub only exposes ACTIONS_ID_TOKEN_REQUEST_URL to jobs that were granted
+# `id-token: write`. Pull requests from forks never are, and neither is a
+# Dependabot PR unless deploy.yml grants it. Those runs skip every AWS step
+# with a notice instead of failing on the OIDC token.
+#
+# Output: available=true|false written to GITHUB_OUTPUT
+
+set -e
+
+if [ -n "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ]; then
+  echo "available=true" >> "${GITHUB_OUTPUT:-/dev/null}"
+  exit 0
+fi
+
+echo "::notice title=Terraform plan skipped::This run has no GitHub OIDC token (pull request from a fork, or a Dependabot PR), so it cannot reach AWS. Terraform is validated and format-checked only; the plan runs after merge, or on a PR from a branch in this repository."
+echo "available=false" >> "${GITHUB_OUTPUT:-/dev/null}"