From 72496b8e557177a4d7375609a78b7b69727e25c7 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Fri, 23 Oct 2026 12:41:38 -0400 Subject: [PATCH 1/2] Protect driver releases with validation and scoped signing Validate the pinned target and effective version before protected approval, then prepare and publish on a fresh runner with short-lived signing sessions. Keep release blockers and the integration matrix ahead of publication, verify all prepared Git signatures, and push branch releases to an explicit ref. Preserve the driver publisher key and document credential migration, consumer verification, dry runs, key rotation, and partial-release recovery. --- .github/workflows/release.yml | 247 +++++----- Makefile | 112 +++-- pom.xml | 4 + scripts/release/README.md | 165 +++++-- scripts/release/guard.py | 617 +++++++++++++++++++++--- scripts/release/signing.py | 406 ++++++++++++---- scripts/release/target.py | 144 ++++++ scripts/release/test_commands.py | 307 ++++++++++++ scripts/release/test_guard.py | 794 +++++++++++++++++++++++++++++-- scripts/release/test_signing.py | 783 ++++++++++++++++++++++++++---- scripts/release/test_target.py | 181 +++++++ 11 files changed, 3250 insertions(+), 510 deletions(-) create mode 100644 scripts/release/target.py create mode 100644 scripts/release/test_commands.py create mode 100644 scripts/release/test_target.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89985d8bdf5..fbc3c1c35a8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ on: skip-tests: type: boolean - description: 'skip-tests: do not run tests while releasing' + description: 'skip-tests: skip project tests; release helper tests always run' default: false target-tag: @@ -22,245 +22,214 @@ on: type: boolean description: 'skip-integration-matrix: do not run the driver matrix integration tests before releasing' default: false + release-version: + type: string + required: false + description: 'Optional four-component 4.x release version override' env: - # The driver line this branch releases; scopes release-blocker issues. - RELEASE_LINE: 4.x - # The other line released from this repository. - OTHER_LINE: 3.x + RELEASE_MAJOR: '4' + RELEASE_TARGET_TAG: ${{ inputs.target-tag }} + RELEASE_VERSION: ${{ inputs.release-version }} + +# Serialize both release lines through the shared publisher environment. +concurrency: + group: java-driver-release + cancel-in-progress: false jobs: - release-target: - name: Resolve release target + validate: + name: Validate release runs-on: ubuntu-latest - permissions: contents: read issues: read - outputs: ref: ${{ steps.target.outputs.ref }} version: ${{ steps.target.outputs.version }} - steps: + - name: Checkout validation source + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Check for open release blockers if: inputs.dry-run == false env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - # Only a blocker scoped solely to the other line is skipped. - blockers="$( - gh issue list \ - --repo "$GITHUB_REPOSITORY" \ - --state open \ - --label release-blocker \ - --limit 100 \ - --json labels,title,url | - jq -r \ - --arg line "area/Driver_-_java-driver-$RELEASE_LINE" \ - --arg other "area/Driver_-_java-driver-$OTHER_LINE" ' - .[] - | [.labels[].name] as $names - | select((any($names[]; . == $other) | not) or any($names[]; . == $line)) - | "\(.url) — \(.title)"' - )" - if [[ -n "$blockers" ]]; then - while IFS= read -r blocker; do - echo "::error::Release blocked by open issue: $blocker" - done <<<"$blockers" - exit 1 - fi - echo "No open release blockers for $RELEASE_LINE." + run: python3 scripts/release/target.py blockers - - name: Checkout Repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - name: Set up validation Java + uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: - fetch-depth: 0 - persist-credentials: false + java-version: '11' + distribution: temurin + + - name: Validate release tooling + run: | + python3 scripts/release/guard.py check + python3 -m unittest discover -s scripts/release -p 'test_*.py' -v + + - name: Preserve validation guard + run: | + mkdir -p "$RUNNER_TEMP/release-safety" + cp scripts/release/*.py scripts/release/publisher.asc Makefile "$RUNNER_TEMP/release-safety/" + echo "RELEASE_GUARD=$RUNNER_TEMP/release-safety/guard.py" >> "$GITHUB_ENV" + echo "RELEASE_LOG_DIR=$RUNNER_TEMP/release-logs" >> "$GITHUB_ENV" - # The commit release:prepare starts from, and the version it will tag: HEAD, or one commit - # before target-tag for a re-release, as make checkout-one-commit-before does. - name: Resolve release commit and version id: target + run: python3 "$RUNNER_TEMP/release-safety/target.py" resolve + + - name: Validate target without release credentials env: - TARGET_TAG: ${{ inputs.target-tag }} + RELEASE_TARGET_COMMIT: ${{ steps.target.outputs.ref }} + RELEASE_VERSION: ${{ steps.target.outputs.version }} + RELEASE_SKIP_TESTS: ${{ inputs.skip-tests }} run: | - set -euo pipefail - ref="$GITHUB_SHA" - if [[ "$TARGET_TAG" == 4.* ]]; then - ref="$(git rev-parse "refs/tags/$TARGET_TAG~1")" - fi - version="$(git show "$ref:pom.xml" | python3 -c 'import sys, xml.etree.ElementTree as ET; print(ET.parse(sys.stdin).getroot().findtext("{http://maven.apache.org/POM/4.0.0}version"))')" - version="${version%-SNAPSHOT}" - if [[ ! "$version" =~ ^4\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::Unexpected release version '$version' at $ref" - exit 1 + python3 "$RUNNER_TEMP/release-safety/target.py" checkout + python3 "$RELEASE_GUARD" check + if [[ "$RELEASE_SKIP_TESTS" == true ]]; then + export MAVEN_OPTS="${MAVEN_OPTS:-} -DskipTests=true -DskipITs=true" fi - if [[ "$TARGET_TAG" == 4.* && "$version" != "$TARGET_TAG" ]]; then - echo "::error::$TARGET_TAG~1 would release $version, not $TARGET_TAG" - exit 1 - fi - echo "Releasing $version from $ref" - echo "ref=$ref" >> "$GITHUB_OUTPUT" - echo "version=$version" >> "$GITHUB_OUTPUT" + python3 "$RELEASE_GUARD" run mvn -B --no-transfer-progress versions:set "-DnewVersion=$RELEASE_VERSION" -DgenerateBackupPoms=false + python3 "$RELEASE_GUARD" run mvn -B --no-transfer-progress xml-format:xml-format + python3 "$RELEASE_GUARD" run mvn -B --no-transfer-progress -Prelease -Dgpg.skip=true verify - - name: Skip integration matrix - if: inputs.skip-integration-matrix == true - run: echo "::warning::Pre-release integration matrix skipped" + - name: Upload sanitized validation logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: validation-maven-logs + path: ${{ runner.temp }}/release-logs/*.log + if-no-files-found: ignore - # The driver matrix patches and tests every release, and a release that breaks it used to - # surface only in the nightly Jenkins run, after it was public (4.19.2.2). This runs the full - # Scylla matrix from matrix master on the commit about to be released, selecting patches as - # if it were already tagged with the release version. If it fails, fix the matrix (for - # example, add versions/scylla/), merge that, and re-dispatch. pre-release-integration: name: Pre-release integration matrix - needs: release-target + needs: validate if: inputs.skip-integration-matrix == false permissions: contents: read uses: scylladb/scylla-java-driver-matrix/.github/workflows/driver-integration-matrix.yml@master with: run_apache: false - driver_ref: ${{ needs.release-target.outputs.ref }} - driver_version: ${{ needs.release-target.outputs.version }} + driver_ref: ${{ needs.validate.outputs.ref }} + driver_version: ${{ needs.validate.outputs.version }} release: name: Release - needs: [release-target, pre-release-integration] - if: ${{ !cancelled() && needs.release-target.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.pre-release-integration.result) }} + needs: [validate, pre-release-integration] + if: ${{ !cancelled() && needs.validate.result == 'success' && (needs.pre-release-integration.result == 'success' || (inputs.skip-integration-matrix && needs.pre-release-integration.result == 'skipped')) }} runs-on: ubuntu-latest - + environment: release permissions: contents: write issues: read - env: - MVNCMD: mvn -B -ntp - + RELEASE_VERSION: ${{ needs.validate.outputs.version }} steps: - - name: Checkout Repository + - name: Checkout workflow launch source uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 - name: Preserve release guard run: | mkdir -p "$RUNNER_TEMP/release-safety" - cp scripts/release/*.py scripts/release/publisher.asc "$RUNNER_TEMP/release-safety/" + cp scripts/release/*.py scripts/release/publisher.asc Makefile "$RUNNER_TEMP/release-safety/" echo "RELEASE_GUARD=$RUNNER_TEMP/release-safety/guard.py" >> "$GITHUB_ENV" echo "RELEASE_LOG_DIR=$RUNNER_TEMP/release-logs" >> "$GITHUB_ENV" - - name: Checkout Code One Commit Before ${{ inputs.target-tag }} - if: inputs.target-tag != 'scylla-4.x' + - name: Checkout release target env: - RELEASE_TARGET_TAG: ${{ inputs.target-tag }} - run: make checkout-one-commit-before + RELEASE_TARGET_COMMIT: ${{ needs.validate.outputs.ref }} + run: python3 "$RUNNER_TEMP/release-safety/target.py" checkout - name: Check release credential safety before importing credentials run: python3 "$RUNNER_TEMP/release-safety/guard.py" check - - name: Set up JDK 11 + - name: Check Central credentials before release preparation + env: + SONATYPE_TOKEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }} + SONATYPE_TOKEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }} + run: python3 "$RUNNER_TEMP/release-safety/target.py" preflight + + - name: Set up publishing Java uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: java-version: '11' - distribution: 'temurin' + distribution: temurin server-id: central - gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} - # Name the env var the release steps export. Otherwise v6 writes - # gpg.passphraseEnvName=GPG_PASSPHRASE and signing fails with "No pinentry" - gpg-passphrase-env-var: MAVEN_GPG_PASSPHRASE server-username: SONATYPE_TOKEN_USERNAME server-password: SONATYPE_TOKEN_PASSWORD - cache: maven - - - name: Configure and check protected release signing - env: - MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: python3 "$RUNNER_TEMP/release-safety/signing.py" configure - name: Prepare release env: + RELEASE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - RELEASE_SKIP_TESTS: ${{ inputs.skip-tests }} - run: make release-prepare + SONATYPE_TOKEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }} + SONATYPE_TOKEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }} + RELEASE_SKIP_TESTS: 'true' + run: | + baseline=$(git rev-parse HEAD) + export RELEASE_BASE_COMMIT="$baseline" + echo "RELEASE_BASE_COMMIT=$baseline" >> "$GITHUB_ENV" + make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-prepare - - name: Verify release commit and tag signatures - env: - RELEASE_VERSION: ${{ needs.release-target.outputs.version }} - run: python3 "$RUNNER_TEMP/release-safety/signing.py" verify-release "$RELEASE_VERSION" + - name: Verify prepared release commits and tag with public key only + run: python3 "$RUNNER_TEMP/release-safety/signing.py" verify-prepared - name: Recheck for open release blockers if: inputs.dry-run == false env: GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - # Only a blocker scoped solely to the other line is skipped. - blockers="$( - gh issue list \ - --repo "$GITHUB_REPOSITORY" \ - --state open \ - --label release-blocker \ - --limit 100 \ - --json labels,title,url | - jq -r \ - --arg line "area/Driver_-_java-driver-$RELEASE_LINE" \ - --arg other "area/Driver_-_java-driver-$OTHER_LINE" ' - .[] - | [.labels[].name] as $names - | select((any($names[]; . == $other) | not) or any($names[]; . == $line)) - | "\(.url) — \(.title)"' - )" - if [[ -n "$blockers" ]]; then - while IFS= read -r blocker; do - echo "::error::Release blocked by open issue: $blocker" - done <<<"$blockers" - exit 1 - fi - echo "No open release blockers for $RELEASE_LINE." + run: python3 "$RUNNER_TEMP/release-safety/target.py" blockers + + - name: Recheck release branch before publication + if: inputs.target-tag == 'scylla-4.x' + env: + RELEASE_TARGET_COMMIT: ${{ needs.validate.outputs.ref }} + run: python3 "$RUNNER_TEMP/release-safety/target.py" check-branch - name: Perform release if: inputs.dry-run == false env: - RELEASE_SKIP_TESTS: ${{ inputs.skip-tests }} + RELEASE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} SONATYPE_TOKEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }} SONATYPE_TOKEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }} - run: make release + RELEASE_SKIP_TESTS: 'true' + run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release - name: Perform release dry-run if: inputs.dry-run == true env: - RELEASE_SKIP_TESTS: ${{ inputs.skip-tests }} + RELEASE_GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} SONATYPE_TOKEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USERNAME }} SONATYPE_TOKEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN_PASSWORD }} - run: make release-dry-run + RELEASE_SKIP_TESTS: 'true' + run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run - name: Push changes to SCM if: ${{ inputs.dry-run == false && inputs.target-tag == 'scylla-4.x' }} - run: | - git status && git log -3 - git push origin --follow-tags -v - - - name: Remove release signing passphrase and configuration - if: always() - run: python3 "$RUNNER_TEMP/release-safety/signing.py" cleanup + run: git push origin --follow-tags -v HEAD:refs/heads/scylla-4.x - name: Upload sanitized release logs if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: maven-stdout + name: maven-logs path: ${{ runner.temp }}/release-logs/*.log if-no-files-found: ignore publish-docs: name: Publish docs needs: release - # Without !cancelled(), a skipped pre-release-integration would skip this job too. if: ${{ !cancelled() && needs.release.result == 'success' && inputs.dry-run == false && inputs.target-tag == 'scylla-4.x' }} - # The release push uses the workflow token, which starts no workflow. uses: ./.github/workflows/docs-pages.yml permissions: contents: write diff --git a/Makefile b/Makefile index 66e087adbc2..46aeb5416f7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,10 @@ RELEASE_GUARD ?= scripts/release/guard.py +RELEASE_TARGET_HELPER ?= $(dir $(RELEASE_GUARD))target.py +RELEASE_SIGNING_HELPER ?= $(dir $(RELEASE_GUARD))signing.py +RELEASE_MAKEFILE := $(abspath $(lastword $(MAKEFILE_LIST))) +export RELEASE_VERSION RELEASE_BASE_COMMIT +.PHONY: release-check-user release-preflight release-prepare release release-dry-run +.PHONY: _release-prepare _release _release-dry-run test-release-signing SHELL := bash .ONESHELL: @@ -299,14 +305,6 @@ resolve-scylla-version: .prepare-get-version echo "$${SCYLLA_VERSION_RESOLVED}" >${SCYLLA_VERSION_FILE} fi -checkout-one-commit-before: - @if [[ "${RELEASE_TARGET_TAG}" == 4.* ]]; then - echo "Checking out one commit before ${RELEASE_TARGET_TAG}" - git fetch --prune --unshallow || git fetch --prune || true - git checkout ${RELEASE_TARGET_TAG}~1 - git tag -d ${RELEASE_TARGET_TAG} - fi - download-cassandra: .prepare-scylla-ccm resolve-cassandra-version @if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then CASSANDRA_VERSION_RESOLVED=$$(cat '${CASSANDRA_VERSION_FILE}') @@ -339,44 +337,90 @@ download-scylla: .prepare-scylla-ccm resolve-scylla-version ccm create ccm_1 -i 127.0.254. -n 1:0 -v "$${SCYLLA_VERSION_RESOLVED}" --scylla --config-dir=/tmp/download.ccm rm -rf /tmp/download.ccm -.require-release-prepare-env: - @if [[ -z "$${MAVEN_GPG_PASSPHRASE}" ]]; then - echo "MAVEN_GPG_PASSPHRASE is empty" - exit 1 +release-check-user: + @set -eu + if [ "$${GITHUB_ACTIONS:-}" = true ]; then + # The release environment authorizes workflow operators independently of the signer. + exit 0 fi - -.require-release-env: - @if [[ -z "$${MAVEN_GPG_PASSPHRASE}" ]]; then - echo "MAVEN_GPG_PASSPHRASE is empty" + if ! user=$$(gh api user --jq .login); then + echo "Release requires GitHub user scylladb-publisher. Run gh auth login or gh auth switch." >&2 exit 1 fi - if [[ -z "$${SONATYPE_TOKEN_USERNAME}" ]]; then - echo "SONATYPE_TOKEN_USERNAME is empty" + if [ "$$user" != scylladb-publisher ]; then + echo "Release requires GitHub user scylladb-publisher (current: $$user)." >&2 exit 1 fi - if [[ -z "$${SONATYPE_TOKEN_PASSWORD}" ]]; then - echo "SONATYPE_TOKEN_PASSWORD is empty" + +release-preflight: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + +release-prepare: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + python3 "$(RELEASE_GUARD)" check + +@python3 "$(RELEASE_SIGNING_HELPER)" exec $(MAKE) -f "$(RELEASE_MAKEFILE)" _release-prepare + +release: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + python3 "$(RELEASE_GUARD)" check + python3 "$(RELEASE_SIGNING_HELPER)" verify-prepared + +@python3 "$(RELEASE_SIGNING_HELPER)" exec $(MAKE) -f "$(RELEASE_MAKEFILE)" _release + +release-dry-run: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + python3 "$(RELEASE_GUARD)" check + python3 "$(RELEASE_SIGNING_HELPER)" verify-prepared + +@python3 "$(RELEASE_SIGNING_HELPER)" exec $(MAKE) -f "$(RELEASE_MAKEFILE)" _release-dry-run + +test-release-signing: + python3 -m unittest discover -s scripts/release -p 'test_signing.py' -v + +_release-prepare: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + if [[ -z "$${MAVEN_GPG_PASSPHRASE}" ]]; then + echo "MAVEN_GPG_PASSPHRASE is empty, can't continue" exit 1 fi - -release-prepare: .require-release-prepare-env - @if [[ "${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "${RELEASE_SKIP_TESTS}" == "1" ]]; then - export MAVEN_OPTS="${MAVEN_OPTS} -DskipTests=true -DskipITs=true" + if [[ "$${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "$${RELEASE_SKIP_TESTS}" == "1" ]]; then + export MAVEN_OPTS="$${MAVEN_OPTS} -DskipTests=true -DskipITs=true" fi - # maven-release-plugin formats the poms via preparationGoals/completionGoals (pom.xml), so the tag passes xml-check. - @python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:prepare -DpushChanges=false + NEXT_RELEASE=() + if [[ -n "$${RELEASE_VERSION}" ]]; then + NEXT_RELEASE=("-DreleaseVersion=$${RELEASE_VERSION}") + fi + export MAVEN_OPTS + baseline=$$(git rev-parse HEAD) + python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:prepare -DpushChanges=false "$${NEXT_RELEASE[@]}" + python3 "$(RELEASE_SIGNING_HELPER)" verify-prepared "$$baseline" -release: .require-release-env - @if [[ "${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "${RELEASE_SKIP_TESTS}" == "1" ]]; then - export MAVEN_OPTS="${MAVEN_OPTS} -DskipTests=true -DskipITs=true" +_release: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + if [[ -z "$${MAVEN_GPG_PASSPHRASE}" ]]; then + echo "MAVEN_GPG_PASSPHRASE is empty, can't continue" + exit 1 + fi + if [[ "$${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "$${RELEASE_SKIP_TESTS}" == "1" ]]; then + export MAVEN_OPTS="$${MAVEN_OPTS} -DskipTests=true -DskipITs=true" fi - @python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:perform -Drelease.autopublish=true + python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:perform -Drelease.autopublish=true -release-dry-run: .require-release-env - @if [[ "${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "${RELEASE_SKIP_TESTS}" == "1" ]]; then - export MAVEN_OPTS="${MAVEN_OPTS} -DskipTests=true -DskipITs=true" +_release-dry-run: release-check-user + @set -eo pipefail + python3 "$(RELEASE_TARGET_HELPER)" preflight + if [[ -z "$${MAVEN_GPG_PASSPHRASE}" ]]; then + echo "MAVEN_GPG_PASSPHRASE is empty, can't continue" + exit 1 + fi + if [[ "$${RELEASE_SKIP_TESTS}" == "true" ]] || [[ "$${RELEASE_SKIP_TESTS}" == "1" ]]; then + export MAVEN_OPTS="$${MAVEN_OPTS} -DskipTests=true -DskipITs=true" fi - @python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:perform + python3 "$(RELEASE_GUARD)" run $(MVNCMD) release:perform compile-all: .install-guava-shaded mvn -B compile test-compile -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true diff --git a/pom.xml b/pom.xml index bc24da460ec..a89b4382bf8 100644 --- a/pom.xml +++ b/pom.xml @@ -987,6 +987,10 @@ height="0" width="0" style="display:none;visibility:hidden"> release + + + true + diff --git a/scripts/release/README.md b/scripts/release/README.md index ecea3c83b36..4769959ddb1 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -1,51 +1,148 @@ -# Release signing +# Driver releases and publisher signing -The release workflow uses the Java driver's repository-specific publisher certificate: +Both release lines follow the +[Alternator release process](https://github.com/scylladb/alternator-client-java/blob/96839d9c1aba6cea23c5065515e6ce1d9a05df8e/.github/workflows/release.yml). +Java APIs, coordinates, module exclusions, and compilation targets are unchanged. + +## Protected environment + +The shared GitHub `release` environment must require `scylladb/drivers` review, permit +self-review, disable administrator bypass, and allow deployment from exactly the branches +`scylla-4.x` and `scylla-3.x`. Dispatch from the corresponding release branch, including +historical releases. Validation and the integration matrix must finish before approval. + +Environment secrets: + +| Secret | Source | +| --- | --- | +| `GPG_PRIVATE_KEY` | Protected `~/.repos-gpg/scylladb/java-driver/signing-private.asc` | +| `GPG_PASSPHRASE` | Matching `~/.repos-gpg/scylladb/java-driver/passphrase.txt` | +| `SONATYPE_TOKEN_USERNAME` | Fresh Central Portal user token username | +| `SONATYPE_TOKEN_PASSWORD` | Fresh Central Portal user token password | + +The initial Central values are `REPLACE_ME`; both must be replaced before a dry run or +release can prepare commits. Generate a token with driver namespace publishing rights +using [Central's token instructions](https://central.sonatype.org/publish/generate-portal-token/). +Upload secret files through stdin (`gh secret set NAME --repo scylladb/java-driver +--env release < FILE`); never place their contents in command arguments or Git. + +Token expiration: **none** (configured during cutover). Keep the full +primary backup and revocation certificate offline. Retain `CODECOV_TOKEN`. + +## Release inputs and validation + +`target-tag` accepts the current release branch or a matching four-component version tag +(`4.MINOR.PATCH.DRIVER` or `3.MINOR.PATCH.DRIVER`). Other refs are rejected. An optional +`release-version` sets the effective version for branch releases. Historical releases +require both the tag POM and its preceding commit's POM to match the tag version; they +cannot use an override to publish under another version. Unsafe historical builds fail +credential-safety checks and must be ported to a supported target before release. + +The workflow resolves the target SHA and version once. It runs release-helper tests and +an unsigned `-Prelease -Dgpg.skip=true verify` at the effective version without release +secrets, then passes that +same SHA and effective version to the driver integration matrix. `skip-tests` skips only +project tests; helper tests always run. The matrix must pass unless +`skip-integration-matrix` is explicitly selected. Open `release-blocker` issues apply to +both lines unless labelled solely for the other line. Blockers are checked before +validation and again immediately before publication, including all result pages. + +Approval starts a fresh runner with no validation workspace or Maven cache. The runner +preserves the launch SHA's Makefile, guard, signing helper, target helper, and public +certificate before checking out the validated SHA. A branch that advanced during approval +is rejected. Publication checks the branch again after preparation. Preparation and +publication skip project tests already covered by validation. The 3.x README is stamped +with the effective version; 4.x retains its POM formatting preparation/completion goals. Its release profile skips +archive assembly: the unpublished distribution archives have no assembly descriptors. + +## Signing and verification + +The driver retains its existing publisher certificate (`publisher.asc`): - Primary: `FA5BF24FD656DB6172B6DEDA92CDA600C75CA820` - Signing subkey: `6633DD12AC5A67675513E2EA998E34B10468CB5B` -- Identity: `ScyllaDB Publisher (scylladb/java-driver) ` - -Both `scylla-4.x` and `scylla-3.x` use this signing subkey for Maven artifacts, release -commits, development-version commits, and annotated release tags. `publisher.asc` is the -public certificate; `signing.py` pins both fingerprints and rejects a different signer. +- Git identity: `ScyllaDB Publisher ` -The existing repository secrets must be updated as a matching pair during activation: +Each signing session imports only the protected signing-subkey export into a fresh +keyring, checks both fingerprints and primary-secret absence, tests the passphrase with +a fresh agent, and signs a preflight payload. Credentials travel through environment and +pipes. Git configuration is scoped to the command and inherited by Maven child checkouts. +Maven artifacts, release/development commits, and annotated tags use the signing subkey. +The session stops children, kills its agent, and deletes its keyring on success, failure, +SIGTERM, SIGHUP, or SIGINT. No private key is imported by `setup-java`. -- `GPG_PRIVATE_KEY`: the protected ASCII-armored **signing-subkey export**, with an - unavailable primary secret, from `signing-private.asc`. -- `GPG_PASSPHRASE`: the corresponding nonempty passphrase. +After preparation, public-only verification checks every commit since the original +`RELEASE_BASE_COMMIT`, requires an annotated signed tag on a newly prepared commit, and +requires the prepared tag to equal the effective release version. Publication rechecks +this history before importing signing credentials. Never substitute a newer baseline to +hide an unsigned or unexpected preparation commit. -Keep the full certification-primary backup and revocation certificate out of CI. Register -the public certificate with `scylladb-publisher` and verify the publisher email there before -creating release commits or tags. Publish the public certificate and fingerprints for -package verification, including the public keyserver required by Maven Central. Keep old -public certificates available to verify existing releases; do not replace existing tags. +Maven output is captured and sanitized before console or artifact output, including +encoded credentials. Both stdout and stderr remain available on failure or cancellation. +Do not enable Maven debug output, redirect raw output, pass credential properties on the +command line, or disable `gpg.bestPractices`. -`setup-java` imports the key into its temporary keyring. Before Maven runs, `signing.py` -checks the exact protected subkey and makes a real test signature with a fresh agent. An -empty or incorrect passphrase stops the job. The script writes a temporary 0600 passphrase -file and creates a Git GPG wrapper using loopback pinentry. A temporary job-specific Git -configuration supplies the publisher identity and signing settings to Maven's separate -`release:perform` checkout too. It does not modify a developer's Git configuration. +Consumers can import the public certificate into an isolated GnuPG keyring and run +`gpg --verify artifact.jar.asc artifact.jar`, `git verify-tag VERSION`, or +`git verify-commit COMMIT`. Independently compare both fingerprints above, including the +signing subkey in GnuPG's `VALIDSIG` status. Keep historical certificates available. -The workflow exports the non-secret `gpg.keyname` selection through `MAVEN_OPTS`, which is -inherited by Maven's forked builds. Maven GPG receives its passphrase through -`MAVEN_GPG_PASSPHRASE`, matching the `setup-java` input. Do not put passphrases in Maven -command-line properties or disable `gpg.bestPractices`. +For local releases, authenticate `gh` as `scylladb-publisher`; protected Actions operators +are authorized by environment review. Export the Central token pair and: -After `release:prepare`, every new commit and the expected release tag are verified in a -public-only keyring, including the exact signing and primary fingerprints. Publication -and SCM push remain behind the existing release gates. The signing passphrase file and -configuration are removed by an `always()` step; `setup-java` cleans up its keyring. +```sh +export RELEASE_GPG_PRIVATE_KEY_FILE="$HOME/.repos-gpg/scylladb/java-driver/signing-private.asc" +export RELEASE_GPG_PASSPHRASE_FILE="$HOME/.repos-gpg/scylladb/java-driver/passphrase.txt" +export RELEASE_VERSION=4.19.2.4 # use the appropriate four-component driver version +export RELEASE_BASE_COMMIT="$(git rev-parse HEAD)" +make release-prepare +python3 scripts/release/signing.py verify-prepared +make release-dry-run +``` -Run the signing and credential-safety tests without production secrets: +Run credential-free helper verification with: ```sh python3 scripts/release/guard.py check python3 -m unittest discover -s scripts/release -p 'test_*.py' -v ``` -The tests create disposable protected keys, commits, tags, and a separate clone. They cover -correct, empty, and incorrect passphrases, rejection of a full primary secret, unsigned -commits, the wrong tag target, foreign signers, and ambiguous verification status. +## Cutover and dry runs + +Pause release dispatches during cutover. Provision the protected environment, merge both +line-specific workflow changes, and replace the Central placeholders. Run an approved +`dry-run` on each branch. Dry runs **upload unpublished Central deployments**; inspect +signatures, coordinates, and module contents in Central, then delete those deployments. +They push no Git refs and publish no docs. See +[Central publishing behavior](https://central.sonatype.org/publish/publish-portal-maven/). + +After both dry runs pass, remove repository-scoped `GPG_PRIVATE_KEY`, `GPG_PASSPHRASE`, +`SONATYPE_TOKEN_USERNAME`, and `SONATYPE_TOKEN_PASSWORD`. Audit all workflow and external +consumers of `OSSRH_USERNAME`/`OSSRH_PASSWORD` before deleting them. Keep `CODECOV_TOKEN`. +Revoke the retired Central token only after checking other repositories and consumers. +Resume release dispatches. Confirm the first real release publishes the expected module +set, signed commits and tag, and docs. Branch releases push explicitly to +`HEAD:refs/heads/scylla-4.x` or `HEAD:refs/heads/scylla-3.x` only after publication succeeds. +Historical targets never push refs or publish docs. + +## Partial release recovery and key rotation + +If validation or approval fails, fix the cause and start a new dispatch. If preparation +fails, inspect sanitized logs and the baseline/commits/tag before retrying; discard only +unpublished local state in a disposable checkout. Never move a published release tag. + +If Central upload/validation fails, inspect the deployment status and module set before +retrying. Delete an unpublished deployment when appropriate. If some artifacts are already +public, Maven Central versions are immutable: do not blindly rerun or overwrite them. +Recover the original prepared commits and tag, verify against the original baseline, and +reconcile the actual Central state first. If publication succeeded but SCM push failed, +verify and push that exact signed history after reconciling branch advancement. Do not +prepare another release under the same version. Run docs publication only after the SCM +release state is correct. Retain sanitized logs and deployment identifiers for recovery. + +For rotation, pause releases, export a new protected signing subkey without the primary +secret, update both pinned fingerprints and the public certificate on both branches, and +replace the environment key/passphrase together. Register the publisher identity on GitHub +and publish the certificate to the public keyserver required by Central. Repeat disposable +key tests and approved dry runs before resuming. Preserve old certificates, published tags, +and revocation records so consumers can verify historical artifacts. diff --git a/scripts/release/guard.py b/scripts/release/guard.py index 83c9e2405c5..460c392f51e 100644 --- a/scripts/release/guard.py +++ b/scripts/release/guard.py @@ -2,22 +2,32 @@ """Check release defaults and capture Maven output without writing raw credentials. Output is buffered until Maven exits (64 MiB maximum), then sanitized for both the -console and log artifacts. Child Maven processes run normally; they are not intercepted. +console and log artifacts. Overflow and incomplete streams produce fixed diagnostics. +Child Maven processes run normally; they are not intercepted. """ import base64 import heapq +import html +import json import os from pathlib import Path import re import selectors import signal +import shlex import subprocess import sys +import time import urllib.parse import xml.etree.ElementTree as ET LIMIT = 64 * 1024 * 1024 +DRAIN_GRACE = 1 +OVERFLOW_LOG = b"Maven output exceeded the capture limit; invocation output withheld.\n" +INCOMPLETE_LOG = b"Maven output stream did not close; incomplete output withheld.\n" +CANCELLED_LOG = b"Maven was interrupted; incomplete output withheld.\n" +FAILED_LOG = b"Sanitized Maven output capture failed; invocation output withheld.\n" ENV_OPTIONS = ( "MAVEN_ARGS", "MAVEN_OPTS", @@ -29,46 +39,209 @@ ) +CHILD_OPTIONS = r"arguments|prepareVerifyArgs|goals|preparationGoals|completionGoals" + + class Unsafe(ValueError): pass def check_text(text, source=None, maven_options=True, maven_arguments=(), jvm_options=False): - if any(arg.strip().startswith("-X") for arg in maven_arguments): - raise Unsafe("Maven debug options are not allowed") + # Maven accepts an equals sign after -D/--D; treat it as the ordinary + # property form before every safety check. + def normalize_property(value): + value = re.sub(r"(?= len(arguments): + check_release_arguments(setting[1]) + if not option_arguments: + for setting in re.finditer( + rf"(?:--?D\s*|--?define(?:=|\s+))(?:{CHILD_OPTIONS})=([^\r\n]*)", + text, re.I, + ): + check_release_arguments(setting[1]) + if any(argument.startswith("@") for argument in arguments + jvm_arguments): + raise Unsafe("unverified argument files are not allowed") + if jvm_options and re.search(r"-Dgpg\.[^\s]*bestPractices[^\s]*[\"']", text, re.I): + raise Unsafe("GPG best practices must not be disabled") + for argument in arguments + jvm_arguments: + check_autopublish(argument) for index, argument in enumerate(maven_arguments): - if index and maven_arguments[index - 1] in ("-D", "-define", "--define"): - argument = maven_arguments[index - 1] + " " + argument + previous = maven_arguments[index - 1].strip().strip("\"'") if index else "" + raw = original_maven_arguments[index].strip() + if len(raw) > 1 and raw[0] in "\"'" and raw[-1] == raw[0]: + raw = raw[1:-1] + if previous in ("-D", "--D", "-define", "--define"): + argument = normalize_property(previous + " " + argument.strip("\"'")) + if re.fullmatch(r"(?:--?D\s*|--?define(?:=|\s+))release\.autopublish=(?:true|false)", + argument, re.I | re.S): + raw_option = (previous + " " if previous in ("-D", "--D", "-define", "--define") + else "") + raw + raw_option = normalize_outer_property(raw_option) + raw_option = re.sub( + r"(?i)^(--?D\s*|--?define(?:=|\s+))([\"'])(.*)\2$", + lambda match: match[1] + match[3], raw_option, + ) + raw_value = raw_option.rsplit("=", 1)[-1] + if raw_value.lower() not in ("true", "false"): + raise Unsafe("release.autopublish must be a Boolean") + # Quote deletion is useful for finding quoted property names, but it + # must not turn a literal value such as tr'ue into a valid Boolean. + if re.fullmatch(r"(?:--?D\s*|--?define(?:=|\s+))gpg\.bestPractices=true", + argument, re.I | re.S): + raw_option = (previous + " " if previous in ("-D", "--D", "-define", "--define") + else "") + raw + value = normalize_outer_property(raw_option).rsplit("=", 1)[-1] + if value.lower() != "true": + raise Unsafe("GPG best practices must not be disabled") + check_autopublish(argument) setting = re.fullmatch( - r"(?:-D\s*|--?define(?:=|\s+))gpg\.bestPractices=(.*)", argument, re.I | re.S + r"(?:--?D\s*|--?define(?:=|\s+))gpg\.bestPractices=(.*)", argument, re.I | re.S ) if setting and setting[1].lower() != "true": raise Unsafe("GPG best practices must not be disabled") + child_setting = re.fullmatch( + rf"(?:--?D\s*|--?define(?:=|\s+))(?:{CHILD_OPTIONS})=(.*)", + argument, re.I | re.S, + ) + if child_setting: + raw_option = (previous + " " if previous in ("-D", "--D", "-define", "--define") + else "") + raw + raw_option = normalize_outer_property(raw_option) + raw_setting = re.fullmatch( + rf"(?:--?D\s*|--?define(?:=|\s+))[\"']?(?:{CHILD_OPTIONS})[\"']?=(.*)", + raw_option, re.I | re.S, + ) + if not raw_setting: + raise Unsafe("malformed release options are not allowed") + check_release_arguments(raw_setting[1]) # Preserve whitespace inside quoted GPG values when checking the full boolean. gpg_text = re.sub( r"([\"'])((?:\\.|(?!\1)[\s\S])*)\1", lambda match: re.sub(r"\s", "\0", match[2]), text, ).translate(str.maketrans("", "", "\"'")) - if re.search(r"(?:-D\s*|--?define(?:=|\s+))gpg\.bestPractices=(?!true(?:\s|$))", + if re.search(r"(?:--?D\s*|--?define(?:=|\s+))gpg\.bestPractices=(?!true(?:\s|$))", gpg_text, re.I) or (jvm_options and re.search( r"(?:^|\s)-Dgpg\.bestPractices(?:\s|$)", gpg_text, re.I)): raise Unsafe("GPG best practices must not be disabled") # Java and Maven option parsing remove quotes from property names and values. - text = text.translate(str.maketrans("", "", "\"'")) + text = normalize_property(text.translate(str.maketrans("", "", "\"'"))) + if re.search(r"(?:--?D\s*|--?define(?:=|\s+))pomFileName\b", text, re.I): + raise Unsafe("alternate Maven POMs are not allowed") + if maven_options and re.search( + r"(?]*=(?:debug|trace)\b" - r"|(?:-D\s*|--?define(?:=|\s+))[\w.-]*" - r"(?:passphrase|password|token|secret|authorization)[\w.-]*=", + r"|simpleLogger[^\s<>]*=[^\s<>]*\$\{" + r"|(?:--?D\s*|--?define(?:=|\s+))[\w.-]*" + r"(?:passphrase|password|token|secret|credentials?|authorization)[\w.-]*=", text, re.I, ) if maven_options and not match: match = re.search( - r"(?:^|[\s=\"'])(?:--log-file(?:[=\s]|$)|-l(?!lr(?:[\s\"']|$)))", text + r"(?:^|[\s=\"'])(?:--log-file(?:[=\s]|$)|-l(?!lr(?:[\s\"']|$))|--l)", text ) if match: if source is not None: @@ -78,6 +251,195 @@ def check_text(text, source=None, maven_options=True, maven_arguments=(), jvm_op raise Unsafe("debug logging or credential arguments are not allowed") +def check_maven_capture(command, require_capture=False, make_variables=False): + # Inspect each shell command separately: a marker in a comment or another + # command does not establish that Maven runs under the guard. + command = re.sub(r"\\\r?\n\s*", " ", command) + if make_variables: + # Only Make expands $(NAME) as a variable. $$(NAME) reaches the shell + # as a command substitution and must not be normalized away. + command = re.sub(r"(?" + index += 1 + command = "".join(characters) + # Check the original script before shlex removes quotes and escapes. Even + # guarded arguments are expanded by the shell before capture starts. + if "$(" in command or "`" in command: + raise Unsafe("unresolved release executables or command substitutions are unsupported") + if grouping: + raise Unsafe("shell grouping and process substitutions are unsupported") + try: + lexer = shlex.shlex(command, posix=True, punctuation_chars=";&|\n") + lexer.whitespace = " \t\r" + lexer.whitespace_split = True + lexer.commenters = "" + tokens = list(lexer) + except ValueError as error: + raise Unsafe("malformed release commands are not allowed") from error + if not require_capture: + # These driver test/build parameters never select a Maven goal. Unknown + # variable goals remain rejected; release recipes allow none of these. + for index, token in enumerate(tokens): + for name in ("MVN_COVERAGE", "MAVEN_EXTRA_ARGS", "SCYLLA_VERSION_RESOLVED", + "CASSANDRA_VERSION_RESOLVED"): + token = token.replace("${" + name + "}", "").replace("$" + name, "") + tokens[index] = token + parts = [[]] + for token in tokens: + if token and all(character in ";&|\n" for character in token): + parts.append([]) + else: + parts[-1].append(token) + for part in parts: + while part and (re.match(r"^[A-Za-z_][A-Za-z_0-9]*=", part[0]) + or part[0] in ("!", "if", "then", "elif", "else", "do", + "exec", "command", "env")): + if part[0] == "env": + part = part[1:] + if part and part[0] == "--": + part = part[1:] + elif part and part[0].startswith("-"): + raise Unsafe("unsupported env launcher options in release commands") + continue + if part[0] in ("command", "exec"): + part = part[1:] + if part and part[0] == "--": + part = part[1:] + elif part and part[0].startswith("-"): + raise Unsafe("unsupported launcher options in release commands") + continue + part = part[1:] + # Resolve neither substitutions nor variable/glob executable names. + if (part and part[0] not in ("$MVNCMD", "${MVNCMD}", "$mvn", "${mvn}", "$MAKE", "${MAKE}", "[", "[[") + and re.search(r"[$`*?\[]", part[0])): + raise Unsafe("unresolved release executables require sanitized log capture") + # Quoted shell scripts are opaque; inspect the executable, not Maven args. + if part and Path(part[0]).name in ( + "sh", "bash", "dash", "ash", "ksh", "zsh", "fish", "eval" + ): + raise Unsafe("nested shell launchers are not allowed in release commands") + if not part: + continue + executable = Path(part[0]).name + if part[0] == "." or executable in ( + "source", "builtin", "xargs", "timeout", "nice", "nohup", "sudo", "doas", + "time", "busybox", "parallel", "watch", "setsid", "stdbuf", "chroot", "flock", "find" + ): + raise Unsafe("unsupported command launchers in release commands") + if part[:3] in (["python3", "$RELEASE_GUARD", "run"], + ["python3", "${RELEASE_GUARD}", "run"]) and len(part) > 3: + continue + if executable in ("echo", "printf"): + continue + is_maven = executable in ("mvn", "mvnw") or part[0] in ("$MVNCMD", "${MVNCMD}", "$mvn", "${mvn}") + # An unknown wrapper can also dispatch Maven. Do not infer safe shell + # semantics from an arbitrary executable's arguments. + if any(Path(token).name in ("mvn", "mvnw") or token in ("$MVNCMD", "${MVNCMD}", "$mvn", "${mvn}") + for token in part[1:]): + raise Unsafe("unsupported command launchers require sanitized log capture") + # Static non-release Maven recipes remain usable without release secrets. + if (is_maven and (require_capture or any("$" in argument for argument in part[1:]))) or any( + "release:" in argument for argument in part + ): + raise Unsafe("release target must use sanitized log capture") + + +def workflow_commands(workflow): + """Read inline and indented run scripts without executing workflow content.""" + lines = iter(workflow.splitlines()) + pending = None + while True: + line = pending if pending is not None else next(lines, None) + pending = None + if line is None: + return + match = re.match(r"^(\s*)(?:-\s+)?(?:run|\"run\"|'run')\s*:\s*(.*)$", line) + if not match: + continue + value = match[2] + if value.startswith(("*", "&", "!")): + raise Unsafe("aliased or tagged release scripts are unsupported") + if not value.startswith(("|", ">")): + if value.startswith(('"', "'")): + # Find the closing YAML quote before accepting a trailing comment. + quoted = re.match( + r'''^("(?:\\.|[^"\\])*"|'(?:''|[^'])*')(?:[ \t]+#.*)?[ \t]*$''', + value, + ) + if not quoted: + raise Unsafe("unsupported quoted release script") + value = quoted[1] + if value.startswith('"'): + try: + value = json.loads(value) + except ValueError as error: + raise Unsafe("unsupported quoted release script") from error + elif value.startswith("'"): + if not value.endswith("'"): + raise Unsafe("unsupported quoted release script") + value = value[1:-1].replace("''", "'") + yield value + continue + script = [] + for following in lines: + if following.strip() and len(following) - len(following.lstrip()) <= len(match[1]): + pending = following + break + script.append(following) + yield (" " if value.startswith(">") else "\n").join(script) + + def check_policy(root): ns = {"m": "http://maven.apache.org/POM/4.0.0"} pom = ET.parse(root / "pom.xml").getroot() @@ -95,11 +457,15 @@ def check_policy(root): or tuple(map(int, version.split("."))) < (0, 11, 0) ): raise Unsafe("update Central publishing before releasing this target") + # Pax provision parameters configure Felix, not Maven's logger/CLI. + provision_params = set(pom.findall( + ".//m:plugin[m:artifactId='maven-pax-plugin']/m:configuration/m:provision/m:param", ns)) for element in pom.iter(): if (element.tag == "{" + ns["m"] + "}gpg.bestPractices" and (element.text or "").strip().lower() != "true"): raise Unsafe("GPG best practices must not be disabled") - check_text(element.text or "") + check_text((element.text or "").replace("${release.autopublish}", "false"), + maven_options=element not in provision_params) paths = [ root / "Makefile", root / ".github/workflows/release.yml", @@ -110,21 +476,46 @@ def check_policy(root): if path.exists(): check_text( path.read_text(), path.relative_to(root), maven_options=path.parent.name == ".mvn", - maven_arguments=( - path.read_text().splitlines() if path.name == "maven.config" else () - ), + maven_arguments=path.read_text().splitlines() if path.name == "maven.config" else (), jvm_options=path.name == "jvm.config", ) - workflow = (root / ".github/workflows/release.yml").read_text() - if "RELEASE_GUARD" not in workflow or "if: always()" not in workflow: - raise Unsafe( - "release workflow sanity check requires RELEASE_GUARD and if: always() markers" - ) + workflow_path = root / ".github/workflows/release.yml" + if not workflow_path.exists(): + raise Unsafe("release workflow .github/workflows/release.yml is missing") + workflow = workflow_path.read_text() + if "RELEASE_GUARD" not in workflow: + raise Unsafe("release workflow requires the RELEASE_GUARD marker") + for command in workflow_commands(workflow): + check_maven_capture(command, require_capture=True) + if not re.search(r"if:\s*(?:always\(\)|\$\{\{\s*always\(\)\s*\}\})", workflow): + raise Unsafe("release workflow requires an always() artifact upload condition") makefile = root / "Makefile" if makefile.exists(): - for line in makefile.read_text().splitlines(): - if re.search(r"\$\(MVNCMD\).*\brelease:", line) and "$(RELEASE_GUARD)" not in line: - raise Unsafe("release target must use sanitized log capture") + logical = re.sub(r"\\\r?\n[ \t]*", " ", makefile.read_text()) + if re.search(r"(?m)^[ \t]*(?:(?:override|export|private)\s+)*\.RECIPEPREFIX\b", logical): + raise Unsafe("custom .RECIPEPREFIX settings are unsupported") + # Existing driver build macros are unrelated to release execution. + checked_directives = logical + for allowed in ["_COVERAGE_NORM := $(or $(shell printf '%s' '$(COVERAGE)' | tr '[:upper:]' '[:lower:]'),false)", "$(error COVERAGE must be one of true/1/yes/on or false/0/no/off, got '$(COVERAGE)')", 'CCM_CONFIG_DIR := $(shell readlink --canonicalize ${CCM_CONFIG_DIR})', 'define LOAD_CACHED_VERSION', 'define REQUIRE_FULLY_QUALIFIED_VERSION']: + checked_directives = checked_directives.replace(allowed + "\n", "\n") + if re.search(r"(?m)^\s*(?:-?include|sinclude|define)\s|\$[({]\s*(?:shell|eval|file|info|warning|error)\b", checked_directives): + raise Unsafe("executable Make directives are unsupported for release targets") + release_target = False + for line in logical.splitlines(): + if line.startswith("\t"): + if release_target or re.search(r"(?:\bmvnw?\b|MVNCMD|\$[({]mvn[)}]|release:)", line): + check_maven_capture(line.lstrip().lstrip("@+-"), require_capture=release_target, make_variables=True) + elif re.match(r"^[^#:=\s][^:=]*:(?!=)", line): + release_target = any( + re.fullmatch(r"[._]?release(?:-[\w-]+)?", target) + for target in line.split(":", 1)[0].split() + ) + prerequisites, separator, recipe = line.split(":", 1)[1].partition(";") + if release_target and prerequisites.strip() not in ("", "release-check-user"): + raise Unsafe("release target prerequisites are unsupported") + if separator: + check_maven_capture(recipe.lstrip().lstrip("@+-"), + require_capture=release_target, make_variables=True) def secret_forms(env): @@ -139,6 +530,15 @@ def secret_forms(env): # Include multiline components, escaped newlines, URL and both Base64 alphabets. values |= {line for value in values for line in value.splitlines() if line} values |= {value.replace("\n", "\\n").replace("\r", "\\r") for value in values} + # JSON/logging serializers also escape quotes, backslashes and Unicode secrets. + values |= {json.dumps(value, ensure_ascii=ascii_only)[1:-1] + for value in tuple(values) for ascii_only in (True, False)} + # Maven XML serializers escape punctuation and may use named or numeric quotes. + for value in tuple(values): + escaped = html.escape(value, quote=False) + for apostrophe in ("'", "'", "'", "'"): + for quote in ('"', """, """, """): + values.add(escaped.replace("'", apostrophe).replace('"', quote)) forms = set(values) for value in values: raw = value.encode() @@ -165,13 +565,11 @@ def spans(pattern, group, flags=0): matches = [ spans( - rb"(?i)([\"'])((?:-D\s*|--?define[=\s]+)[\w.-]*" - rb"(?:passphrase|password|username|token|secret|authorization)[\w.-]*=)" + rb"(?i)([\"'])((?:-D\s*|--?define[=\s]+)[\w.-]*(?:passphrase|password|username|token|secret|authorization)[\w.-]*=)" rb"((?:\\[\s\S]|(?!\1)[^\\])*)\1", 3, ), spans( - rb"(?i)((?:-D\s*|--?define[=\s]+)[\w.-]*" - rb"(?:passphrase|password|username|token|secret|authorization)[\w.-]*=)" + rb"(?i)((?:-D\s*|--?define[=\s]+)[\w.-]*(?:passphrase|password|username|token|secret|authorization)[\w.-]*=)" rb"(\"(?:\\[\s\S]|[^\"\\])*\"|'(?:\\[\s\S]|[^'\\])*'|[^\s]+)", 2, ), spans(rb"(?im)((?:proxy-)?authorization[\"']?\s*[:=]\s*)([^\r\n]*)", 2), @@ -194,61 +592,113 @@ def spans(pattern, group, flags=0): return bytes(output) +def write_logs(clean, logs): + """Write only sanitized content or fixed diagnostics, tolerating broken outputs.""" + output_failed = False + for name, data, stream in zip(("stdout", "stderr"), clean, (sys.stdout, sys.stderr)): + try: + logs.mkdir(parents=True, exist_ok=True) + with (logs / (name + ".log")).open("ab") as output: + output.write(data) + except OSError: + output_failed = True + try: + stream.buffer.write(data) + stream.flush() + except OSError: + output_failed = True + # Prevent a failed shutdown flush from replacing Maven's exit status. + try: + with open(os.devnull, "wb") as sink: + os.dup2(sink.fileno(), stream.fileno()) + except OSError: + pass + if output_failed: + try: + os.write(2, b"Sanitized release output could not be fully written; values withheld.\n") + except OSError: + pass + + def capture(command, env, forms, logs): - """Never spool raw output. Limit memory and terminate the process group on failure.""" + """Never spool raw output or sanitize a stream cut off before EOF.""" process = subprocess.Popen( command, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True ) buffers = [bytearray(), bytearray()] + complete = [False, False] + overflow = False + logs_started = False try: with selectors.DefaultSelector() as selector: selector.register(process.stdout, selectors.EVENT_READ, 0) selector.register(process.stderr, selectors.EVENT_READ, 1) + exited_at = None while selector.get_map(): - for key, _ in selector.select(): + if process.poll() is not None: + if exited_at is None: + exited_at = time.monotonic() + elif time.monotonic() - exited_at >= DRAIN_GRACE: + break + for key, _ in selector.select(timeout=0.1): chunk = os.read(key.fileobj.fileno(), 65536) if not chunk: + complete[key.data] = True selector.unregister(key.fileobj) - else: + elif not overflow: + if sum(map(len, buffers)) + len(chunk) > LIMIT: + overflow = True + buffers = [bytearray(), bytearray()] + continue buffers[key.data].extend(chunk) - if sum(map(len, buffers)) > LIMIT: - raise Unsafe("release output exceeds memory limit") status = process.wait() # Sanitize both streams completely before opening artifacts or emitting output. - clean = [redact(bytes(buffer), forms) for buffer in buffers] - output_failed = False - for name, data, stream in zip(("stdout", "stderr"), clean, (sys.stdout, sys.stderr)): - try: - logs.mkdir(parents=True, exist_ok=True) - with (logs / (name + ".log")).open("ab") as output: - output.write(data) - except OSError: - output_failed = True - try: - stream.buffer.write(data) - stream.flush() - except OSError: - output_failed = True - # Prevent a failed shutdown flush from replacing Maven's exit status. - try: - with open(os.devnull, "wb") as sink: - os.dup2(sink.fileno(), stream.fileno()) - except OSError: - pass - if output_failed: - try: - os.write( - 2, b"Sanitized release output could not be fully written; values withheld.\n" - ) - except OSError: - pass + try: + clean = [OVERFLOW_LOG, OVERFLOW_LOG] if overflow else [ + redact(bytes(buffer), forms) if ended else INCOMPLETE_LOG + for buffer, ended in zip(buffers, complete) + ] + except Exception: + # Publishing has already exited. Withhold output without changing its + # status or skipping the workflow's successful SCM push. + clean = [FAILED_LOG, FAILED_LOG] + logs_started = True + write_logs(clean, logs) return status if status >= 0 else 128 - status - except BaseException: + except BaseException as error: + completed_status = process.poll() + signum = signal.SIGTERM + if isinstance(error, SystemExit) and isinstance(error.code, int): + signum = error.code - 128 + if signum not in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + signum = signal.SIGTERM + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + pass + try: + process.wait(timeout=DRAIN_GRACE) + except subprocess.TimeoutExpired: + pass + # The Maven parent may exit before a descendant responds to cancellation. try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: pass process.wait() + clean = [FAILED_LOG, FAILED_LOG] + if isinstance(error, (SystemExit, KeyboardInterrupt)): + try: + clean = [OVERFLOW_LOG, OVERFLOW_LOG] if overflow else [ + redact(bytes(buffer), forms) if ended else CANCELLED_LOG + for buffer, ended in zip(buffers, complete) + ] + except Exception: + pass + if not logs_started: + write_logs(clean, logs) + if isinstance(error, Exception) and completed_status is not None: + return completed_status if completed_status >= 0 else 128 - completed_status raise finally: process.stdout.close() @@ -259,31 +709,36 @@ def interrupted(signum, frame): raise SystemExit(128 + signum) +def check_runtime_options(): + for name in ENV_OPTIONS: + check_text( + os.environ.get(name, ""), + maven_arguments=os.environ.get(name, "").split() if name == "MAVEN_ARGS" else (), + jvm_options=name not in ("MAVEN_ARGS", "MAVEN_CONFIG"), + ) + for name in ("maven.config", "jvm.config"): + path = Path(".mvn") / name + if path.exists(): + check_text( + path.read_text(), + maven_arguments=path.read_text().splitlines() if name == "maven.config" else (), + jvm_options=name == "jvm.config", + ) + + def main(): try: if sys.argv[1:] == ["check"]: + check_runtime_options() check_policy(Path.cwd()) print("Release credential safety checks passed.") return 0 - if sys.argv[1:3] != ["run", "mvn"]: - raise Unsafe("expected check or run mvn ...") + if len(sys.argv) < 3 or sys.argv[1] != "run" or not sys.argv[2]: + raise Unsafe("expected check or run Maven executable ...") if any(arg.startswith("-X") for arg in sys.argv[3:]): raise Unsafe("Maven debug options are not allowed") check_text(" ".join(sys.argv[3:]), maven_arguments=sys.argv[3:]) - for name in ENV_OPTIONS: - check_text( - os.environ.get(name, ""), - maven_arguments=os.environ.get(name, "").split() if name == "MAVEN_ARGS" else (), - jvm_options=name not in ("MAVEN_ARGS", "MAVEN_CONFIG"), - ) - for name in ("maven.config", "jvm.config"): - path = Path(".mvn") / name - if path.exists(): - check_text( - path.read_text(), - maven_arguments=path.read_text().splitlines() if name == "maven.config" else (), - jvm_options=name == "jvm.config", - ) + check_runtime_options() for signum in (signal.SIGTERM, signal.SIGHUP): signal.signal(signum, interrupted) return capture( @@ -292,6 +747,10 @@ def main(): secret_forms(os.environ), Path(os.environ.get("RELEASE_LOG_DIR", "/tmp/release-logs")), ) + except Unsafe as error: + # Unsafe messages name only fixed policy checks, never their input values. + print(f"Release safety check failed: {error}; values withheld.", file=sys.stderr) + return 2 except Exception: # Exception values can contain credentials. Do not echo them. print( diff --git a/scripts/release/signing.py b/scripts/release/signing.py index 01fd3db4918..72a2bb77059 100644 --- a/scripts/release/signing.py +++ b/scripts/release/signing.py @@ -1,28 +1,64 @@ #!/usr/bin/env python3 """Configure and verify the repository's protected release signing subkey.""" +from contextlib import contextmanager import os from pathlib import Path -import shutil +import shlex +import signal import subprocess import sys import tempfile -import uuid +import time PRIMARY = "FA5BF24FD656DB6172B6DEDA92CDA600C75CA820" SIGNING = "6633DD12AC5A67675513E2EA998E34B10468CB5B" PUBLIC_KEY = Path(__file__).with_name("publisher.asc") +TERMINATION_GRACE = 2 class SigningError(ValueError): pass -def run(command, env=None): - result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=60) +def command_operation(command): + # Only fixed labels enter diagnostics; argv and child output may contain secrets. + if command[0] == "gpg": + for flag, label in [("--import", "key import"), ("--detach-sign", "signature creation"), + ("--verify", "signature verification"), + ("--list-secret-keys", "secret-key inspection")]: + if flag in command: + return "GPG " + label + return "GPG operation" + if command[0] == "gpgconf": + return "GPG agent shutdown" + if command[0] == "git": + for subcommand in ["verify-commit", "verify-tag", "check-ref-format", "cat-file", "rev-parse"]: + if subcommand in command: + return "Git " + subcommand + return "Git operation" + return "command" + + +def invoke(command, **kwargs): + operation = command_operation(command) + try: + return subprocess.run(command, **kwargs) + except subprocess.TimeoutExpired: + raise SigningError(operation + " timed out.") from None + except OSError: + raise SigningError(operation + " could not start (OS error).") from None + + +def run(command, env=None, **kwargs): + result = invoke(command, env=env, capture_output=True, text=True, timeout=60, **kwargs) if result.returncode: - raise SigningError("Release signing command failed; values withheld.") - return result.stdout + result.stderr + raise SigningError(f"{command_operation(command)} failed (exit status {result.returncode}).") + # Git writes --raw verification status to stderr; ordinary callers parse stdout. + if command[0] == "git" and "--raw" in command and any( + operation in command for operation in ("verify-commit", "verify-tag")): + return result.stdout + result.stderr + return result.stdout def gpg(*args, env=None): @@ -31,23 +67,28 @@ def gpg(*args, env=None): def verify_status(status): - signatures = [line.split() for line in status.splitlines() - if line.startswith("[GNUPG:] VALIDSIG ")] + records = [line.split() for line in status.splitlines() if line.startswith("[GNUPG:] ")] + signatures = [row for row in records if len(row) > 1 and row[1] == "VALIDSIG"] + good = [row for row in records if len(row) > 1 and row[1] == "GOODSIG"] + rejected = {"EXPKEYSIG", "REVKEYSIG", "EXPSIG", "BADSIG", "ERRSIG", "FAILURE", "ERROR", + "KEYEXPIRED", "KEYREVOKED", "SIGEXPIRED"} if (len(signatures) != 1 or len(signatures[0]) != 12 - or signatures[0][2] != SIGNING or signatures[0][-1] != PRIMARY): + or signatures[0][2] != SIGNING or signatures[0][-1] != PRIMARY + or len(good) != 1 or len(good[0]) < 3 or good[0][2] != SIGNING[-16:] + or any(len(row) > 1 and row[1] in rejected for row in records)): raise SigningError("Signature does not match the repository's release subkey.") -def protected_subkey(): +def protected_subkey(env): rows = [line.split(":") for line in - gpg("--with-colons", "--with-keygrip", "--list-secret-keys").splitlines()] + gpg("--with-colons", "--with-keygrip", "--list-secret-keys", env=env).splitlines()] fingerprints = [row[9] for row in rows if row[0] == "fpr"] keys = [row for row in rows if row[0] in ("sec", "ssb")] grips = [row[9] for row in rows if row[0] == "grp"] if (fingerprints != [PRIMARY, SIGNING] or len(keys) != 2 or len(grips) != 2 or keys[0][14] != "#" or keys[1][14] != "+"): raise SigningError("Import only this repository's signing-private.asc release subkey.") - keydir = Path(os.environ["GNUPGHOME"]) / "private-keys-v1.d" + keydir = Path(env["GNUPGHOME"]) / "private-keys-v1.d" if ((keydir / (grips[0] + ".key")).exists() or b"(protected-private-key" not in (keydir / (grips[1] + ".key")).read_bytes()): raise SigningError("Release signing requires a protected subkey without the primary secret.") @@ -60,106 +101,285 @@ def write_private(path, text, executable=False): path.chmod(0o700) -def public_environment(directory): - return {**os.environ, "GNUPGHOME": str(directory / "public")} +@contextmanager +def public_environment(): + with tempfile.TemporaryDirectory(prefix="release-public-") as temp: + env = {**os.environ, "GNUPGHOME": temp} + try: + gpg("--import", PUBLIC_KEY, env=env) + yield env + finally: + run(["gpgconf", "--homedir", temp, "--kill", "gpg-agent"]) + +def passphrase_value(env): + value = env.get("RELEASE_GPG_PASSPHRASE", + env.get("MAVEN_GPG_PASSPHRASE", env.get("PGP_PASSPHRASE", ""))) + if "RELEASE_GPG_PASSPHRASE_FILE" in env: + value = Path(env["RELEASE_GPG_PASSPHRASE_FILE"]).read_text().removesuffix("\n") + if not value or "\n" in value or "\r" in value or len(value.encode()) > 1024: + raise SigningError("Release signing requires a nonempty single-line passphrase.") + return value -def configure(): - passphrase = os.environ.get("MAVEN_GPG_PASSPHRASE", "") - if not passphrase or "\n" in passphrase or "\r" in passphrase: - raise SigningError("MAVEN_GPG_PASSPHRASE must contain a nonempty single-line passphrase.") - protected_subkey() - directory = Path(tempfile.mkdtemp(prefix="release-signing-", dir=os.environ["RUNNER_TEMP"])) + +def credential_gpg(args, payload=None, capture=True, env=None): + env = os.environ if env is None else env + # A separate pipe preserves stdin for Git's payload. Its descriptor, never + # its contents, appears in argv; pass_fds deliberately survives only to gpg. + reader, writer = os.pipe() + try: + try: + os.write(writer, (passphrase_value(env) + "\n").encode()) + finally: + os.close(writer) + command = ["gpg", "--no-options", "--batch", "--no-tty", "--no-auto-key-retrieve", + "--pinentry-mode", "loopback", "--passphrase-fd", str(reader), *map(str, args)] + if capture: + return run(command, env=env, input=payload, pass_fds=(reader,)) + return invoke(command, input=payload, pass_fds=(reader,), env=env, timeout=60).returncode + finally: + os.close(reader) + + +class SessionSignals: + def __init__(self): + self.signum = None + + def __call__(self, signum, frame): + # Recording cannot interrupt resource acquisition or teardown midway. + if self.signum is None: + self.signum = signum + + def check(self): + if self.signum is not None: + raise SystemExit(128 + self.signum) + + +@contextmanager +def session_signals(): + signals = (signal.SIGTERM, signal.SIGHUP, signal.SIGINT) + previous = {signum: signal.getsignal(signum) for signum in signals} + current = previous[signal.SIGTERM] + if isinstance(current, SessionSignals) and all(handler is current for handler in previous.values()): + yield current + return + interrupted = SessionSignals() try: - passphrase_file = directory / "passphrase" - write_private(passphrase_file, passphrase + "\n") - wrapper = directory / "gpg-wrapper" - write_private(wrapper, '#!/bin/sh\nexec gpg --no-options --batch --no-tty ' - '--pinentry-mode loopback --passphrase-file ' - '"$RELEASE_GPG_PASSPHRASE_FILE" "$@"\n', executable=True) - public = directory / "public" - public.mkdir(mode=0o700) - verify_env = public_environment(directory) - gpg("--import", PUBLIC_KEY, env=verify_env) - artifact = directory / "preflight" - write_private(artifact, "ScyllaDB Java driver release signing preflight\n") - # Kill the dedicated signing agent: cached credentials cannot satisfy this check. - run(["gpgconf", "--homedir", os.environ["GNUPGHOME"], "--kill", "gpg-agent"]) - gpg("--pinentry-mode", "loopback", "--passphrase-file", passphrase_file, - "--local-user", SIGNING + "!", "--detach-sign", artifact) - verify_status(gpg("--status-fd", "1", "--verify", str(artifact) + ".sig", artifact, - env=verify_env)) - config = directory / "gitconfig" - # A job-specific global file is inherited by Maven's localCheckout clones too. - write_private(config, "") - settings = {"user.name": "ScyllaDB Publisher", "user.email": "publish.code@scylladb.com", - "user.signingkey": SIGNING + "!", "commit.gpgsign": "true", - "tag.gpgSign": "true", "gpg.format": "openpgp", "gpg.program": str(wrapper)} - for name, value in settings.items(): - run(["git", "config", "--file", str(config), name, value]) - values = { - "RELEASE_SIGNING_DIR": str(directory), - "RELEASE_GPG_PASSPHRASE_FILE": str(passphrase_file), - "GIT_CONFIG_GLOBAL": str(config), - "MAVEN_OPTS": (os.environ.get("MAVEN_OPTS", "") - + " -Dgpg.keyname=" + SIGNING + "!").strip(), - "RELEASE_BASE_COMMIT": run(["git", "rev-parse", "HEAD"]).strip(), - } - with Path(os.environ["GITHUB_ENV"]).open("a") as output: - for name, value in values.items(): - marker = "SIGNING_" + uuid.uuid4().hex - output.write(f"{name}<<{marker}\n{value}\n{marker}\n") - return values - except BaseException: - shutil.rmtree(directory) - raise - - -def verify_release(tag): - directory = Path(os.environ["RELEASE_SIGNING_DIR"]) - env = public_environment(directory) - baseline = os.environ["RELEASE_BASE_COMMIT"] + for signum in signals: + signal.signal(signum, interrupted) + yield interrupted + finally: + for signum, handler in previous.items(): + signal.signal(signum, handler) + interrupted.check() + + +def execute(command, env): + with session_signals() as interrupted: + process = None + try: + interrupted.check() + try: + process = subprocess.Popen(command, env=env, start_new_session=True) + except OSError: + raise SigningError(command_operation(command) + " could not start (OS error).") from None + while True: + interrupted.check() + try: + status = process.wait(timeout=0.1) + except subprocess.TimeoutExpired: + continue + interrupted.check() + return status if status >= 0 else 128 - status + except BaseException as error: + if process is None: + raise + signum = error.code - 128 if isinstance(error, SystemExit) and isinstance(error.code, int) else signal.SIGTERM + if signum not in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + signum = signal.SIGTERM + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + process.wait() + raise error + deadline = time.monotonic() + TERMINATION_GRACE + try: + process.wait(timeout=TERMINATION_GRACE) + except subprocess.TimeoutExpired: + pass + # Make may exit before guard finishes stopping its separate Maven group. + time.sleep(max(0, deadline - time.monotonic())) + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + raise + + +@contextmanager +def signing_session(): + env = dict(os.environ) + passphrase_value(env) + private = env.pop("RELEASE_GPG_PRIVATE_KEY", "") + if "RELEASE_GPG_PRIVATE_KEY_FILE" in env: + private = Path(env.pop("RELEASE_GPG_PRIVATE_KEY_FILE")).read_text() + if not private: + raise SigningError("Release signing requires a protected signing-subkey export.") + # Recursive Make can restore consumed credentials and replace the session's + # Maven options from serialized command-line overrides. Ordinary overrides + # already exist in env; keep execution flags without re-exporting that list. + env.pop("MAKEOVERRIDES", None) + if "MAKEFLAGS" in env: + env["MAKEFLAGS"] = (" " + env["MAKEFLAGS"]).split(" -- ", 1)[0][1:] + with session_signals() as interrupted, tempfile.TemporaryDirectory(prefix="release-signing-", dir=env.get("RUNNER_TEMP")) as temp: + directory = Path(temp) + signing_home = directory / "keyring" + signing_home.mkdir(mode=0o700) + env["GNUPGHOME"] = str(signing_home) + try: + credential_gpg(["--import"], payload=private, env=env) + interrupted.check() + protected_subkey(env) + wrapper = directory / "gpg-wrapper" + write_private(wrapper, '#!/bin/sh\nexec python3 ' + + shlex.quote(str(Path(__file__).resolve())) + + ' gpg-program "$@"\n', executable=True) + # Configuration is inherited only by the command and its children. + # Runtime entries override checkout settings without changing files. + settings = {"user.name": "ScyllaDB Publisher", "user.email": "publish.code@scylladb.com", + "user.signingkey": SIGNING + "!", "commit.gpgsign": "true", + "tag.gpgSign": "true", "gpg.format": "openpgp", "gpg.program": str(wrapper)} + count = int(env.get("GIT_CONFIG_COUNT", "0")) + for index, (name, value) in enumerate(settings.items(), start=count): + env[f"GIT_CONFIG_KEY_{index}"] = name + env[f"GIT_CONFIG_VALUE_{index}"] = value + env["GIT_CONFIG_COUNT"] = str(count + len(settings)) + # Maven's forked release builds inherit these public JVM options and + # the passphrase environment input supported by maven-gpg-plugin. + env["MAVEN_OPTS"] = (env.get("MAVEN_OPTS", "") + + " -Dgpg.keyname=" + SIGNING + "!").strip() + env["MAVEN_GPG_PASSPHRASE"] = passphrase_value(env) + artifact = directory / "preflight" + write_private(artifact, "Repository release signing preflight\n") + # Cached credentials must not satisfy the passphrase check. + run(["gpgconf", "--homedir", str(signing_home), "--kill", "gpg-agent"]) + credential_gpg(["--local-user", SIGNING + "!", "--detach-sign", artifact], env=env) + # First use materializes openpgp-native imports. Before then even an + # unprotected export can appear as a protected-private-key to the agent. + protected_subkey(env) + with public_environment() as verify_env: + verify_status(gpg("--status-fd", "1", "--verify", str(artifact) + ".sig", artifact, + env=verify_env)) + interrupted.check() + yield env + finally: + run(["gpgconf", "--homedir", str(signing_home), "--kill", "gpg-agent"]) + + +def verify_git(kind, ref): + with public_environment() as env: + verify_status(run(["git", "-c", "gpg.program=gpg", "verify-" + kind, "--raw", ref], env)) + + +def verify_release(tag, baseline=None): + baseline = baseline or os.environ.get("RELEASE_BASE_COMMIT") + if not baseline: + raise SigningError("Release verification requires the pre-prepare RELEASE_BASE_COMMIT or an explicit baseline.") run(["git", "merge-base", "--is-ancestor", baseline, "HEAD"]) commits = run(["git", "rev-list", baseline + "..HEAD"]).splitlines() if not commits: - raise SigningError("Maven did not create any release commits.") - for commit in commits: - verify_status(run(["git", "-c", "gpg.program=gpg", "verify-commit", "--raw", commit], env)) + raise SigningError("No release commits were created.") + with public_environment() as env: + for commit in commits: + verify_status(run(["git", "-c", "gpg.program=gpg", "verify-commit", "--raw", commit], env)) + ref = "refs/tags/" + tag + run(["git", "check-ref-format", ref]) + if run(["git", "cat-file", "-t", ref]).strip() != "tag": + raise SigningError("Release tags must be signed annotated tags.") + if run(["git", "rev-parse", ref + "^{commit}"]).strip() not in commits: + raise SigningError("Release tag does not point to a newly signed release commit.") + verify_status(run(["git", "-c", "gpg.program=gpg", "verify-tag", "--raw", ref], env)) + + +def verify_prepared(baseline=None): + properties = Path("release.properties").read_text().splitlines() + tags = [line.removeprefix("scm.tag=") for line in properties if line.startswith("scm.tag=")] + if len(tags) != 1: + raise SigningError("Expected exactly one prepared Maven release tag.") + expected = os.environ.get("RELEASE_VERSION") + if expected and tags[0] != expected: + raise SigningError("Prepared Maven tag does not match the validated release version.") + verify_release(tags[0], baseline) + + +def verify_commit(commit): + verify_git("commit", commit) + + +def verify_tag(tag, commit=None): ref = "refs/tags/" + tag run(["git", "check-ref-format", ref]) - if run(["git", "rev-parse", ref + "^{commit}"]).strip() not in commits: - raise SigningError("Release tag does not point to a newly signed release commit.") - verify_status(run(["git", "-c", "gpg.program=gpg", "verify-tag", "--raw", ref], env)) + if run(["git", "cat-file", "-t", ref]).strip() != "tag": + raise SigningError("Release tags must be signed annotated tags.") + if commit is not None and run(["git", "rev-parse", ref + "^{commit}"]).strip() != commit: + raise SigningError("Release tag points to an unexpected commit.") + verify_git("tag", ref) -def cleanup(): - value = os.environ.get("RELEASE_SIGNING_DIR") - if not value: - return - directory = Path(value).resolve() - if (directory.parent != Path(os.environ["RUNNER_TEMP"]).resolve() - or not directory.name.startswith("release-signing-")): - raise SigningError("Unexpected release signing directory.") - if directory.exists(): - shutil.rmtree(directory) +def sign_files(files, signing_env=None): + if not files: + raise SigningError("At least one release artifact is required.") + if (signing_env is None + and (os.environ.get("RELEASE_GPG_PRIVATE_KEY") or "RELEASE_GPG_PRIVATE_KEY_FILE" in os.environ) + and any(not Path(str(Path(filename).resolve()) + ".asc").exists() for filename in files)): + with signing_session() as session: + return sign_files(files, session) + with session_signals() as interrupted, public_environment() as env: + for filename in files: + interrupted.check() + artifact = Path(filename).resolve() + if not artifact.is_file() or artifact.suffix == ".asc": + raise SigningError("Expected an unsigned release artifact.") + # Existing signatures are immutable: never overwrite them on a rerun. + signature = Path(str(artifact) + ".asc") + if not signature.exists(): + credential_gpg(["--local-user", SIGNING + "!", "--armor", + "--detach-sign", "--output", signature, artifact], env=signing_env) + interrupted.check() + verify_status(gpg("--status-fd", "1", "--verify", signature, artifact, env=env)) def main(): try: - if sys.argv[1:] == ["configure"]: - configure() - elif len(sys.argv) == 3 and sys.argv[1] == "verify-release": - verify_release(sys.argv[2]) - elif sys.argv[1:] == ["cleanup"]: - cleanup() + if len(sys.argv) > 1 and sys.argv[1] == "gpg-program": + # Git requires stdout/stderr byte-for-byte; do not print a success banner. + return credential_gpg(sys.argv[2:], capture=False) + if len(sys.argv) in (2, 3) and sys.argv[1] == "verify-prepared": + verify_prepared(*sys.argv[2:]) + elif len(sys.argv) in (3, 4) and sys.argv[1] == "verify-release": + verify_release(*sys.argv[2:]) + elif len(sys.argv) == 3 and sys.argv[1] == "verify-commit": + verify_commit(sys.argv[2]) + elif len(sys.argv) in (3, 4) and sys.argv[1] == "verify-tag": + verify_tag(*sys.argv[2:]) + elif len(sys.argv) > 2 and sys.argv[1] == "sign-files": + sign_files(sys.argv[2:]) + elif len(sys.argv) > 2 and sys.argv[1] == "exec": + with signing_session() as env: + return execute(sys.argv[2:], env) else: - raise SigningError("Expected configure, verify-release TAG, or cleanup.") + raise SigningError("Expected verify-prepared [BASELINE], verify-release TAG [BASELINE], verify-commit COMMIT, verify-tag TAG [COMMIT], sign-files FILE..., or exec COMMAND...") print("Release signing " + sys.argv[1] + " passed.") return 0 except SigningError as error: print(str(error), file=sys.stderr) - except Exception: - print("Release signing failed; values withheld.", file=sys.stderr) + except Exception as error: + category = ("OS error" if isinstance(error, OSError) else + "invalid value" if isinstance(error, ValueError) else "internal error") + operation = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] in ( + "gpg-program", "verify-prepared", "verify-release", "verify-commit", "verify-tag", "sign-files", "exec") else "command" + print(f"Release signing {operation} failed ({category}); values withheld.", file=sys.stderr) return 1 diff --git a/scripts/release/target.py b/scripts/release/target.py new file mode 100644 index 00000000000..22b97f88c85 --- /dev/null +++ b/scripts/release/target.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Resolve immutable driver release inputs and enforce the two release-line policies.""" + +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + + +class Invalid(ValueError): + pass + + +def git(*args): + return subprocess.check_output(['git', *args], text=True, stderr=subprocess.PIPE).strip() + + +def version(major, value): + if not re.fullmatch(rf'{major}\.[0-9]+\.[0-9]+\.[0-9]+', value): + raise Invalid('Expected a four-component release version for this driver line.') + return value + + +def pom_version(commit): + pom = ET.fromstring(git('show', commit + ':pom.xml')) + return pom.findtext('{http://maven.apache.org/POM/4.0.0}version') or '' + + +def resolve(major, target, override, launch_sha, launch_ref): + branch = f'scylla-{major}.x' + if launch_ref != 'refs/heads/' + branch: + raise Invalid('Dispatch this workflow from its release branch.') + if target == branch: + commit = git('rev-parse', launch_sha + '^{commit}') + else: + version(major, target) + commit = git('rev-parse', 'refs/tags/' + target + '^1') + effective = version(major, override or pom_version(commit).removesuffix('-SNAPSHOT')) + if target != branch and (effective != target or + pom_version(commit).removesuffix('-SNAPSHOT') != target or + pom_version('refs/tags/' + target).removesuffix('-SNAPSHOT') != target): + raise Invalid('Historical target and preparation POM must match the tag version.') + return commit, effective + + +def checkout(major, target, commit): + if not re.fullmatch('[0-9a-f]{40}', commit): + raise Invalid('Invalid validated commit.') + branch = f'scylla-{major}.x' + if target != branch: + version(major, target) + git('fetch', '--no-tags', 'origin', commit) + if target == branch: + git('fetch', '--no-tags', 'origin', 'refs/heads/' + branch) + if git('rev-parse', 'FETCH_HEAD') != commit: + raise Invalid('Release branch advanced after validation; dispatch again.') + git('checkout', '-B', branch, commit) + else: + git('checkout', '--detach', commit) + # release:prepare creates this tag again, locally only for historical targets. + if git('tag', '--list', target): + git('tag', '-d', target) + + +def preflight(): + override = os.environ.get('RELEASE_VERSION', '') + if override: + current = ET.parse('pom.xml').getroot().findtext( + '{http://maven.apache.org/POM/4.0.0}version') or '' + version(current.split('.')[0], override) + for name in ('SONATYPE_TOKEN_USERNAME', 'SONATYPE_TOKEN_PASSWORD'): + value = os.environ.get(name, '') + if not value.strip() or value.strip() == 'REPLACE_ME': + raise Invalid(name + ' must contain a fresh Central Portal token, not a placeholder.') + + +def blockers(major): + # Paginate: a relevant blocker beyond the first 100 must still stop publication. + payload = subprocess.check_output([ + 'gh', 'api', '--paginate', '--slurp', + f'repos/{os.environ["GITHUB_REPOSITORY"]}/issues?state=open&labels=release-blocker&per_page=100', + ], text=True) + other = '3' if major == '4' else '4' + matches = [] + for page in json.loads(payload): + for issue in page: + if 'pull_request' in issue: + continue + labels = {item['name'] for item in issue['labels']} + if (f'area/Driver_-_java-driver-{other}.x' not in labels or + f'area/Driver_-_java-driver-{major}.x' in labels): + matches.append(issue['html_url']) + if matches: + raise Invalid('Release blocked by open issues: ' + ', '.join(matches)) + + +def stamp_readme(): + release = version('3', os.environ['RELEASE_VERSION']) + path = Path('README.md') + text, count = re.subn(r'(?\n ' + executable + "\n clean # RELEASE_GUARD", + 'python3 "$RELEASE_GUARD" run $MVNCMD clean; ' + executable + ' release:clean', + '|\n python3 "$RELEASE_GUARD" run $MVNCMD clean\n ' + + executable + ' release:clean', + ): + workflow_path.write_text(workflow.replace(original, "run: " + script)) + with self.subTest(script=script), self.assertRaises(guard.Unsafe): + guard.check_policy(root) + workflow_path.write_text(workflow.replace( + original, 'run: python3 "${RELEASE_GUARD}" run ' + executable + " clean" + )) + guard.check_policy(root) + workflow_path.write_text(workflow.replace( + original, 'run: >\n python3 "$RELEASE_GUARD" run\n $MVNCMD clean' + )) + guard.check_policy(root) + + def test_unresolved_executables_require_guard_routing(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + workflow_path = root / ".github/workflows/release.yml" + workflow = workflow_path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + for executable in ('$(printf mvn)', '"$(printf mvn)"', '`printf mvn`', + '$TOOL', '${TOOLS}/mvn', './mv*'): + for prefix in ('', 'SAFE=true ', 'exec ', 'env '): + command = prefix + executable + ' release:perform' + with self.subTest(command=command): + (root / "Makefile").write_text('release:\n\t' + command + '\n') + with self.assertRaisesRegex(guard.Unsafe, 'unresolved release executables'): + guard.check_policy(root) + (root / "Makefile").unlink() + workflow_path.write_text(workflow.replace(original, 'run: |\n ' + command)) + with self.assertRaisesRegex(guard.Unsafe, 'unresolved release executables'): + guard.check_policy(root) + workflow_path.write_text(workflow) + guarded = 'SAFE=true python3 "$RELEASE_GUARD" run ' + executable + ' release:perform' + if "$(" in executable or "`" in executable: + # Shell expansion happens before the guard can capture it. + with self.assertRaises(guard.Unsafe): + guard.check_maven_capture(guarded, require_capture=True) + else: + guard.check_maven_capture(guarded, require_capture=True) + for executable in ('/usr/bin/mvn', './mvnw', 'custom-maven'): + guard.check_maven_capture( + 'python3 "$RELEASE_GUARD" run ' + executable + ' release:perform', + require_capture=True, + ) + # Real workflow includes multiline gh command substitutions in assignments. + guard.check_policy(root) + + def test_workflow_quoted_and_spaced_run_keys(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + path = root / ".github/workflows/release.yml" + workflow = path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + for key in ('run :', '"run":', "'run':", '"run" :', "'run' :"): + for value in ('mvn release:perform', '"mvn release:perform"', + '|\n mvn release:perform', + '>\n mvn\n release:perform'): + with self.subTest(key=key, value=value): + path.write_text(workflow.replace(original, key + ' ' + value)) + with self.assertRaisesRegex(guard.Unsafe, 'sanitized log capture'): + guard.check_policy(root) + path.write_text(workflow.replace('run:', key)) + guard.check_policy(root) + self.assertEqual(list(guard.workflow_commands('- ' + key + ' mvn clean')), + ['mvn clean']) + + def test_assignment_substitutions_and_eval_cannot_bypass_capture(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + for command in ('result=$$(mvn release:perform)', + 'result="$$(mvn release:perform)"', + 'result=`mvn release:perform`', + "eval 'mvn release:perform'", + "command eval 'mvn release:perform'", + "env -- eval 'mvn release:perform'"): + with self.subTest(command=command): + (root / "Makefile").write_text('release:\n\t' + command + '\n') + with self.assertRaises(guard.Unsafe): + guard.check_policy(root) + + def test_blocker_lookup_allows_only_existing_fixed_command(self): + script = "blocker=$(gh issue list --repo \"$GITHUB_REPOSITORY\" --state open --label release-blocker --limit 1 --json title,url --jq '.[0] // empty')" + guard.check_maven_capture(script, require_capture=True) + for extra in ('; mvn release:perform', ' $(mvn release:perform)', + ' `mvn release:perform`', '\n mvn release:perform'): + with self.subTest(extra=extra), self.assertRaises(guard.Unsafe): + guard.check_maven_capture(script.replace('gh issue list', 'gh issue list' + extra), + require_capture=True) + + def test_env_launcher_options_cannot_hide_executables(self): + for command in ('env -- "$$(command -v mvn)" -B release:perform', + 'env -- SAFE=true "$$(command -v mvn)" release:perform', + 'env -- mvn release:perform', + 'env -- /bin/bash -c "mvn release:perform"', + 'env -S "mvn release:perform"', + 'env --split-string="mvn release:perform"'): + with self.subTest(command=command), self.assertRaises(guard.Unsafe): + guard.check_maven_capture(command, require_capture=True) + guard.check_maven_capture( + 'env -- SAFE=true python3 "$RELEASE_GUARD" run mvn release:perform', + require_capture=True, + ) + + def test_guarded_maven_arguments_are_not_shell_launchers(self): + for argument in ('-Dsafe.path=/tmp/bash', '/tmp/bash', 'eval', 'sh'): + with self.subTest(argument=argument): + guard.check_maven_capture( + 'python3 "$RELEASE_GUARD" run mvn release:perform ' + argument, + require_capture=True, make_variables=True, + ) + + def test_quoted_workflow_run_comments_preserve_hashes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + path = root / '.github/workflows/release.yml' + workflow = path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + for command in ('python3 "$RELEASE_GUARD" run mvn clean -Dsafe.path=/tmp/#bash', + "python3 '$RELEASE_GUARD' run mvn clean -Dsafe.path=/tmp/#bash", + 'mvn release:perform # RELEASE_GUARD'): + for scalar in (json.dumps(command), "'" + command.replace("'", "''") + "'"): + value = scalar + ' # YAML comment with "quotes" and # hashes' + with self.subTest(value=value): + self.assertEqual(list(guard.workflow_commands('run: ' + value)), [command]) + path.write_text(workflow.replace(original, 'run: ' + value)) + if command.startswith('mvn'): + with self.assertRaisesRegex(guard.Unsafe, 'sanitized log capture'): + guard.check_policy(root) + else: + guard.check_policy(root) + + def test_make_rejects_custom_recipe_prefix(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + for setting in ('.RECIPEPREFIX := >', '.RECIPEPREFIX = >', + 'override .RECIPEPREFIX := >', 'export .RECIPEPREFIX := >'): + with self.subTest(setting=setting): + (root / "Makefile").write_text(setting + '\nrelease:\n>mvn release:perform\n') + with self.assertRaisesRegex(guard.Unsafe, r'\.RECIPEPREFIX.*unsupported'): + guard.check_policy(root) + (root / "Makefile").write_text('# .RECIPEPREFIX := >\nrelease:\n\t@echo safe\n') + guard.check_policy(root) + + def test_make_rejects_nested_shell_launchers(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + for launcher in ("bash -c", "/bin/sh -c", "env bash -lc", "exec zsh -c"): + for executable in ("mvn", "$(MVNCMD)", "$$MVNCMD"): + for target in ("compile", ".release", "release-prepare"): + for separator in ("\n\t", " ; "): + (root / "Makefile").write_text( + target + ":" + separator + "@" + launcher + " '" + + executable + " -B release:perform' # $(RELEASE_GUARD)\n" + ) + with self.subTest(launcher=launcher, executable=executable, + target=target, separator=separator), \ + self.assertRaisesRegex(guard.Unsafe, "nested shell launchers"): + guard.check_policy(root) + + def test_workflow_rejects_nested_shell_launchers(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + workflow_path = root / ".github/workflows/release.yml" + workflow_path.parent.mkdir(parents=True) + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + workflow = (ROOT / ".github/workflows/release.yml").read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + self.assertIn(original, workflow) + for launcher in ("sh", "bash", "dash", "ash", "ksh", "zsh", "fish", "/bin/bash"): + for script in ("mvn -B release:perform", "$MVNCMD clean"): + command = launcher + " -c '" + script + "' # RELEASE_GUARD" + for value in (command, json.dumps(command), "|\n " + command): + workflow_path.write_text(workflow.replace(original, "run: " + value)) + with self.subTest(launcher=launcher, script=script, value=value), \ + self.assertRaisesRegex(guard.Unsafe, "nested shell launchers"): + guard.check_policy(root) + + def test_make_inline_recipes_require_capture(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + for target in (".release", "release-prepare", "compile", "compile .release"): + for executable in ("mvn", "$(MVNCMD)", "${MVNCMD}"): + for arguments in ("-B release:perform", "$(GOAL)", "clean"): + for prefix in ("", "@", "+", "-", "echo capture; "): + (root / "Makefile").write_text( + "GOAL = release:perform\n" + target + ": ; " + + prefix + executable + " " + arguments + + " # $(RELEASE_GUARD)\n" + ) + with self.subTest(target=target, executable=executable, + arguments=arguments, prefix=prefix): + if target == "compile" and arguments == "clean": + guard.check_policy(root) + else: + with self.assertRaises(guard.Unsafe): + guard.check_policy(root) + (root / "Makefile").write_text( + "compile: ; " + executable + " compile test-compile -Dfmt.skip=true\n" + + target + ': ; @python3 "$(RELEASE_GUARD)" run ' + executable + + " -B \\\n\t release:perform\n" + ) + guard.check_policy(root) + + def test_release_prerequisites_and_executable_make_directives_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + for text in ( + 'release: helper\nhelper:\n\tbash -c "mvn release:perform"\n', + 'include unsafe.mk\nrelease:\n\techo safe\n', + 'X := $(shell mvn release:perform)\nrelease:\n\techo safe\n', + '$(eval release: ; mvn release:perform)\n', + 'release:\n\tcustom-maven release:perform\n', + ): + with self.subTest(text=text): + (root / "Makefile").write_text(text) + with self.assertRaises(guard.Unsafe): + guard.check_policy(root) + + def test_argument_substitutions_and_opaque_launchers_fail_full_policy(self): + commands = ( + 'echo "$(mvn release:perform)"', + 'printf "%s\\n" "$(mvn release:perform)"', + 'echo "`mvn release:perform`"', + 'echo "$(mvn)"', + 'python3 "$RELEASE_GUARD" run mvn "$(mvn release:perform)"', + 'python3 "$RELEASE_GUARD" run mvn "`mvn release:perform`"', + "builtin eval 'mvn release:perform'", + "command -p bash -c 'mvn release:perform'", + "command -- builtin eval 'mvn release:perform'", + "exec -a safe bash -c 'mvn release:perform'", + "env -S 'mvn release:perform'", + 'xargs mvn clean', + 'xargs custom-maven release:perform', + 'timeout 10 mvn clean', + 'nice mvn clean', + 'nohup mvn clean', + 'custom-wrapper mvn clean', + 'custom-maven release:perform', + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + workflow_path = root / ".github/workflows/release.yml" + workflow = workflow_path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + self.assertIn(original, workflow) + for command in commands: + with self.subTest(command=command, source="workflow"): + workflow_path.write_text(workflow.replace(original, 'run: ' + json.dumps(command))) + with self.assertRaises(guard.Unsafe): + guard.check_policy(root) + workflow_path.write_text(workflow) + # Make escapes shell dollars; the rejection must survive that layer. + for target in ('release', 'compile'): + with self.subTest(command=command, source="Makefile", target=target): + (root / "Makefile").write_text( + target + ':\n\t' + command.replace('$', '$$') + '\n' + ) + with self.assertRaises(guard.Unsafe): + guard.check_policy(root) + (root / "Makefile").unlink() + + def test_diagnostic_arguments_and_supported_launchers_pass_full_policy(self): + commands = ( + 'echo mvn release:perform', + 'printf "%s\\n" mvn release:perform', + 'echo bash eval custom-maven release:perform', + 'command -- echo mvn release:perform', + 'env -- SAFE=true printf "%s\\n" mvn release:perform', + 'command -- python3 "$RELEASE_GUARD" run custom-maven release:perform', + 'exec -- python3 "$RELEASE_GUARD" run mvn release:perform', + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + workflow_path = root / ".github/workflows/release.yml" + workflow = workflow_path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + for command in commands: + with self.subTest(command=command): + workflow_path.write_text(workflow.replace(original, 'run: ' + json.dumps(command))) + (root / "Makefile").write_text('release:\n\t' + command + '\n') + guard.check_policy(root) + + def test_workflow_aliases_and_tags_are_rejected_before_credentials(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + path = root / ".github/workflows/release.yml" + workflow = path.read_text() + for value in ("*release", "&release mvn release:perform", "!!str mvn release:perform"): + with self.subTest(value=value): + path.write_text("command: &release mvn release:perform\n" + + workflow.replace('run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run', "run: " + value)) + with self.assertRaisesRegex(guard.Unsafe, "aliased or tagged"): + guard.check_policy(root) + + def test_process_substitutions_and_subshells_fail_full_policy(self): + commands = ( + 'cat <(mvn deploy)', + 'cat >(/usr/bin/mvn deploy)', + 'python3 "$RELEASE_GUARD" run mvn clean <(mvn deploy)', + 'python3 "$RELEASE_GUARD" run mvn clean >(mvn deploy)', + '(mvn $GOAL)', + '( mvn $GOAL )', + 'echo safe; (mvn $GOAL)', + 'if true; then (mvn $GOAL); fi', + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / '.github', root / '.github') + shutil.copy(ROOT / 'pom.xml', root / 'pom.xml') + workflow_path = root / '.github/workflows/release.yml' + workflow = workflow_path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + self.assertIn(original, workflow) + for command in commands: + with self.subTest(command=command, source='workflow'): + workflow_path.write_text(workflow.replace(original, 'run: ' + json.dumps(command))) + with self.assertRaisesRegex(guard.Unsafe, 'grouping and process substitutions'): + guard.check_policy(root) + workflow_path.write_text(workflow) + with self.subTest(command=command, source='Makefile'): + (root / 'Makefile').write_text( + 'GOAL = release:perform\n.release:\n\t' + + command.replace('$GOAL', '$(GOAL)') + '\n' + ) + with self.assertRaisesRegex(guard.Unsafe, 'grouping and process substitutions'): + guard.check_policy(root) + (root / 'Makefile').unlink() + + def test_shell_comments_preserve_command_boundaries_full_policy(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / '.github', root / '.github') + shutil.copy(ROOT / 'pom.xml', root / 'pom.xml') + workflow_path = root / '.github/workflows/release.yml' + workflow = workflow_path.read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + self.assertIn(original, workflow) + for executable in ('mvn', '/usr/bin/mvn', './mvnw', '$MVNCMD'): + for comment in ('# captured', '# ignored (mvn deploy) <(mvn deploy)'): + script = ('python3 "$RELEASE_GUARD" run $MVNCMD clean ' + + comment + '\n' + executable + ' release:perform') + with self.subTest(executable=executable, comment=comment): + workflow_path.write_text(workflow.replace( + original, 'run: |\n ' + script.replace('\n', '\n ') + )) + with self.assertRaisesRegex(guard.Unsafe, 'sanitized log capture'): + guard.check_policy(root) + + def test_comments_and_literal_parentheses_pass_full_policy(self): + commands = ( + 'python3 "$RELEASE_GUARD" run mvn clean # captured', + '# ignored (mvn deploy) <(mvn deploy) >(mvn deploy) $(mvn deploy)\n' + 'python3 "$RELEASE_GUARD" run mvn clean', + 'python3 "$RELEASE_GUARD" run mvn clean # captured\necho safe # done', + 'python3 "$RELEASE_GUARD" run mvn clean "-Dlabel=(ordinary)"', + "python3 \"$RELEASE_GUARD\" run mvn clean '-Dlabel=<(ordinary)>(text)'", + r'python3 "$RELEASE_GUARD" run mvn clean -Dlabel=\(ordinary\)', + 'python3 "$RELEASE_GUARD" run mvn clean -Dlabel=hash#text', + 'python3 "$RELEASE_GUARD" run mvn clean "-Dlabel=#(text)"', + 'if test -n "$GOAL"; then python3 "$RELEASE_GUARD" run mvn clean; fi', + 'if [[ -n "$GOAL" ]]; then python3 "$RELEASE_GUARD" run mvn clean; fi', + ) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / '.github', root / '.github') + shutil.copy(ROOT / 'pom.xml', root / 'pom.xml') + workflow_path = root / '.github/workflows/release.yml' + workflow = workflow_path.read_text() + makefile = (ROOT / 'Makefile').read_text() + original = 'run: make -f "$RUNNER_TEMP/release-safety/Makefile" RELEASE_GUARD="$RUNNER_TEMP/release-safety/guard.py" release-dry-run' + self.assertIn(original, workflow) + for command in commands: + with self.subTest(command=command): + workflow_path.write_text(workflow.replace(original, 'run: ' + json.dumps(command))) + # Keep all native credential gates while adding a safe recipe. + (root / 'Makefile').write_text( + makefile + '\nrelease-lexical:\n\t' + + command.replace('$', '$$').replace('\n', '\n\t') + '\n' + ) + guard.check_policy(root) + def test_old_release_target(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -252,6 +790,29 @@ def test_newer_central_versions(self): with self.subTest(version=version): guard.check_policy(root) + def test_workflow_policy_diagnostics(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + shutil.copytree(ROOT / ".github", root / ".github") + shutil.copy(ROOT / "pom.xml", root / "pom.xml") + workflow = root / ".github/workflows/release.yml" + original = workflow.read_text() + workflow.write_text(original.replace("if: always()", "if: ${{ always() }}")) + guard.check_policy(root) + for text, message in ((original.replace("RELEASE_GUARD", "GUARD"), "RELEASE_GUARD"), + (original.replace("if: always()", "if: success()"), "always()")): + workflow.write_text(text) + with self.assertRaisesRegex(guard.Unsafe, re.escape(message)): + guard.check_policy(root) + workflow.unlink() + with self.assertRaisesRegex(guard.Unsafe, "workflow.*missing"): + guard.check_policy(root) + with patch.object(sys, "argv", ["guard.py", "check"]), patch.object( + guard, "check_policy", side_effect=guard.Unsafe("release workflow requires RELEASE_GUARD") + ), patch("sys.stderr", io.StringIO()) as output: + self.assertEqual(guard.main(), 2) + self.assertIn("RELEASE_GUARD", output.getvalue()) + def test_option_locations_hide_values(self): for text in ("safe\ncurl -X POST secret", "safe\n-X", "safe\n--define\npassword=secret"): with patch("sys.stderr", new_callable=io.StringIO) as output: @@ -269,13 +830,15 @@ def test_termination_signals_stop_maven(self): ready = root / "ready" mvn = root / "mvn" mvn.write_text(f"#!{sys.executable}\nimport os, time\n" + "os.write(1, b'-Dpassword=\"first\\nsecret tail')\n" f"open({str(ready) + '.tmp'!r}, 'w').write(str(os.getpid()))\n" f"os.rename({str(ready) + '.tmp'!r}, {str(ready)!r})\n" "time.sleep(30)\n") mvn.chmod(0o755) process = subprocess.Popen( [sys.executable, str(ROOT / "scripts/release/guard.py"), "run", "mvn"], - env=dict(os.environ, PATH=directory + os.pathsep + os.environ["PATH"]), + env=dict(os.environ, PATH=directory + os.pathsep + os.environ["PATH"], + RELEASE_LOG_DIR=str(root / "logs")), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) try: @@ -285,8 +848,11 @@ def test_termination_signals_stop_maven(self): self.assertTrue(ready.exists()) child = int(ready.read_text()) process.send_signal(signum) - process.communicate(timeout=5) + stdout, stderr = process.communicate(timeout=5) self.assertEqual(process.returncode, 128 + signum) + for name, console in (("stdout", stdout), ("stderr", stderr)): + self.assertEqual(console, guard.CANCELLED_LOG) + self.assertEqual((root / "logs" / (name + ".log")).read_bytes(), console) with self.assertRaises(ProcessLookupError): os.kill(child, 0) finally: @@ -319,6 +885,41 @@ def test_encoded_and_multiline_credentials(self): b"Authorization: [REDACTED]\n-Dpassword=[REDACTED]", ) + def test_xml_escaped_credentials_in_both_streams(self): + secret = "xml&\"'secret" + forms = ( + "xml&<tag>\"'secret", + "xml&<tag>"'secret", + "xml&<tag>"'secret", + "xml&<tag>"'secret", + "xml&<tag>"'secret", + ) + output = ("\n".join(forms) + "\n").encode() + with tempfile.TemporaryDirectory() as directory: + command = [sys.executable, "-c", + f"import os; os.write(1, {output!r}); os.write(2, {output!r})"] + script = ( + "import guard, os\nfrom pathlib import Path\n" + f"raise SystemExit(guard.capture({command!r}, os.environ, " + f"guard.secret_forms(os.environ), Path({directory!r})))" + ) + result = subprocess.run( + [sys.executable, "-c", script], capture_output=True, + cwd=ROOT / "scripts/release", + env=dict(os.environ, MAVEN_GPG_PASSPHRASE=secret), + ) + self.assertEqual(result.returncode, 0) + for name, console in (("stdout", result.stdout), ("stderr", result.stderr)): + self.assertEqual(console, b"[REDACTED]\n" * len(forms)) + self.assertEqual((Path(directory) / (name + ".log")).read_bytes(), console) + + def test_quoted_authorization_headers(self): + for header in ("Authorization", "Proxy-Authorization"): + for quote in ("", "'", '"'): + with self.subTest(header=header, quote=quote): + data = f"{quote}{header}{quote}: {quote}Bearer unknown{quote}\n".encode() + self.assertNotIn(b"unknown", guard.redact(data, [])) + def test_quoted_credential_arguments(self): for quote in (b"'", b'"'): for option in (b"-Dpassword=", b"--define=token=", b"--define secret=", @@ -359,17 +960,6 @@ def test_overlapping_structured_and_literal_secrets(self): self.assertEqual(result, b"[REDACTED]") self.assertEqual(guard.redact(b"ababa", [b"aba"]), b"[REDACTED]") - def test_quoted_authorization_headers(self): - for header in ( - b'Authorization: Bearer unknown', - b'"Authorization": "Bearer unknown"', - b"'Proxy-Authorization': 'Basic unknown'", - ): - with self.subTest(header=header): - self.assertEqual( - guard.redact(header, []), header.split(b":", 1)[0] + b": [REDACTED]" - ) - def test_split_output_artifacts_and_exit_status(self): with tempfile.TemporaryDirectory() as directory: # Use a real subprocess to verify both console streams and the resulting files. @@ -396,27 +986,42 @@ def test_split_output_artifacts_and_exit_status(self): self.assertEqual(console, (Path(directory) / (name + ".log")).read_bytes()) self.assertEqual(console, b"diagnostic: [REDACTED]\n") + def test_capture_read_failure_preserves_completed_maven_status(self): + for status in (0, 17): + with self.subTest(status=status), tempfile.TemporaryDirectory() as directory: + process = unittest.mock.Mock() + process.poll.return_value = status + process.wait.return_value = status + with patch.object(guard.subprocess, "Popen", return_value=process), \ + patch.object(guard.selectors, "DefaultSelector", side_effect=OSError("secret")), \ + patch.object(guard.os, "killpg"), patch.object(guard, "write_logs") as logs: + self.assertEqual(guard.capture(["mvn"], {}, [], Path(directory)), status) + logs.assert_called_once_with([guard.FAILED_LOG, guard.FAILED_LOG], Path(directory)) + + def test_configured_maven_executable(self): + for executable in ("/usr/bin/mvn", "./mvnw", "custom-maven"): + with self.subTest(executable=executable), \ + patch.object(sys, "argv", ["guard.py", "run", executable, "release:prepare"]), \ + patch.object(guard, "capture", return_value=0) as capture, \ + patch.object(guard, "check_runtime_options"): + self.assertEqual(guard.main(), 0) + self.assertEqual(capture.call_args.args[0], [executable, "release:prepare"]) + def test_output_failures_preserve_maven_status(self): for status in (0, 17): for failure in ("directory", "artifact", "stdout", "stderr"): - with self.subTest( - status=status, failure=failure - ), tempfile.TemporaryDirectory() as directory: + with self.subTest(status=status, failure=failure), tempfile.TemporaryDirectory() as directory: logs = Path(directory) / "logs" if failure == "directory": logs.write_text("blocked") elif failure == "artifact": logs.mkdir() (logs / "stdout.log").mkdir() - command = [ - sys.executable, "-c", - "import os; os.write(1, b'out\\n'); os.write(2, b'err\\n'); " - f"raise SystemExit({status})", - ] + command = [sys.executable, "-c", + f"import os; os.write(1, b'out\\n'); os.write(2, b'err\\n'); raise SystemExit({status})"] script = ( "import guard, os\nfrom pathlib import Path\n" - f"raise SystemExit(guard.capture({command!r}, os.environ, [], " - f"Path({str(logs)!r})))" + f"raise SystemExit(guard.capture({command!r}, os.environ, [], Path({str(logs)!r})))" ) read_fd, write_fd = os.pipe() os.close(read_fd) @@ -439,16 +1044,111 @@ def test_output_failures_preserve_maven_status(self): if failure != "directory": self.assertEqual((logs / "stderr.log").read_bytes(), b"err\n") - def test_sanitization_and_memory_failures_write_no_logs(self): - for failure in ( - patch.object(guard, "redact", side_effect=ValueError("secret")), - patch.object(guard, "LIMIT", 1), - ): - with tempfile.TemporaryDirectory() as directory, failure: - logs = Path(directory) / "logs" - with self.assertRaises((ValueError, guard.Unsafe)): - guard.capture([sys.executable, "-c", 'print("secret")'], os.environ, [], logs) - self.assertFalse(logs.exists()) + def test_sanitization_failure_writes_fixed_logs(self): + for status in (0, 17): + with self.subTest(status=status), tempfile.TemporaryDirectory() as directory: + script = ( + "import guard, os\nfrom pathlib import Path\n" + "guard.redact = lambda *_: (_ for _ in ()).throw(ValueError('secret'))\n" + f"raise SystemExit(guard.capture([{sys.executable!r}, '-c', " + f"'print(\"secret\"); raise SystemExit({status})'], os.environ, [], Path({directory!r})))" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + cwd=ROOT / "scripts/release") + self.assertEqual(result.returncode, status) + self.assertEqual(result.stdout, guard.FAILED_LOG) + self.assertEqual(result.stderr, guard.FAILED_LOG) + self.assertEqual((Path(directory) / "stdout.log").read_bytes(), guard.FAILED_LOG) + self.assertEqual((Path(directory) / "stderr.log").read_bytes(), guard.FAILED_LOG) + + def test_overflow_drains_without_interrupting_maven(self): + for status in (0, 17): + with self.subTest(status=status), tempfile.TemporaryDirectory() as directory: + marker = Path(directory) / "completed" + command = [sys.executable, "-c", + "import os\nfor fd in (1, 2):\n" + " os.write(fd, b'-Dpassword=\"first\\nsecret tail' * 100)\n" + f"open({str(marker)!r}, 'w').write('done')\nraise SystemExit({status})"] + script = ( + "import guard, os\nfrom pathlib import Path\nguard.LIMIT = 64\n" + f"raise SystemExit(guard.capture({command!r}, os.environ, [], Path({directory!r})))" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + cwd=ROOT / "scripts/release", timeout=5) + self.assertEqual(result.returncode, status) + self.assertEqual(marker.read_text(), "done") + for name, console in (("stdout", result.stdout), ("stderr", result.stderr)): + self.assertEqual(console, guard.OVERFLOW_LOG) + self.assertEqual((Path(directory) / (name + ".log")).read_bytes(), console) + + def test_interruption_during_log_writes_does_not_replay_output(self): + for signum in (signal.SIGTERM, signal.SIGHUP): + for partial in (False, True): + with self.subTest(signum=signum, partial=partial), tempfile.TemporaryDirectory() as directory: + logs = Path(directory) / "logs" + calls = Path(directory) / "writes" + command = [sys.executable, "-c", + "import os; os.write(1, b'out\\n'); os.write(2, b'err\\n')"] + script = ( + "import guard, os\nfrom pathlib import Path\n" + "original = guard.write_logs\n" + "def interrupted_write(clean, logs):\n" + f" with Path({str(calls)!r}).open('a') as out: out.write('called\\n')\n" + + (" logs.mkdir(parents=True, exist_ok=True)\n" + " with (logs / 'stdout.log').open('ab') as out: out.write(clean[0])\n" + if partial else " original(clean, logs)\n") + + f" guard.interrupted({int(signum)}, None)\n" + "guard.write_logs = interrupted_write\n" + f"raise SystemExit(guard.capture({command!r}, os.environ, [], Path({str(logs)!r})))" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + cwd=ROOT / "scripts/release", timeout=5) + self.assertEqual(result.returncode, 128 + signum) + self.assertEqual(calls.read_text(), "called\n") + self.assertEqual((logs / "stdout.log").read_bytes(), b"out\n") + if partial: + self.assertFalse((logs / "stderr.log").exists()) + else: + self.assertEqual(result.stdout, b"out\n") + self.assertEqual(result.stderr, b"err\n") + self.assertEqual((logs / "stderr.log").read_bytes(), b"err\n") + + def test_inherited_pipes_have_bounded_safe_capture(self): + for complete_stderr in (False, True): + with self.subTest(complete_stderr=complete_stderr), tempfile.TemporaryDirectory() as directory: + pid_file = Path(directory) / "child" + command = [sys.executable, "-c", + "import os, time\nchild = os.fork()\nif child == 0:\n" + + (" os.close(2)\n" if complete_stderr else "") + + " os.write(1, b'known-secret-prefix')\n" + " time.sleep(30)\n os._exit(0)\n" + f"open({str(pid_file)!r}, 'w').write(str(child))\n" + + ("os.write(2, b'-Dpassword=\"first\\nunknown secret tail\"')\n" + if complete_stderr else "os.write(2, b'-Dpassword=\"first\\nunknown secret tail')\n") + + "raise SystemExit(17)"] + script = ( + "import guard, os\nfrom pathlib import Path\n" + f"raise SystemExit(guard.capture({command!r}, os.environ, [b'known-secret-prefix-tail'], Path({directory!r})))" + ) + started = time.monotonic() + try: + result = subprocess.run([sys.executable, "-c", script], capture_output=True, + cwd=ROOT / "scripts/release", timeout=5) + self.assertLess(time.monotonic() - started, 4) + self.assertEqual(result.returncode, 17) + self.assertEqual(result.stdout, guard.INCOMPLETE_LOG) + self.assertEqual((Path(directory) / "stdout.log").read_bytes(), result.stdout) + if complete_stderr: + self.assertEqual(result.stderr, b'-Dpassword=[REDACTED]') + else: + self.assertEqual(result.stderr, guard.INCOMPLETE_LOG) + self.assertEqual((Path(directory) / "stderr.log").read_bytes(), result.stderr) + finally: + if pid_file.exists(): + try: + os.kill(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass if __name__ == "__main__": diff --git a/scripts/release/test_signing.py b/scripts/release/test_signing.py index b508e4d8ef0..580f8eedb1d 100644 --- a/scripts/release/test_signing.py +++ b/scripts/release/test_signing.py @@ -1,22 +1,31 @@ """Exercise release signing with disposable protected keys and real Git objects.""" -from contextlib import contextmanager +from contextlib import contextmanager, redirect_stderr +from io import StringIO +import json import os from pathlib import Path import shutil +import signal import subprocess +import sys import tempfile +import time import unittest from unittest.mock import patch import signing +ROOT = Path(__file__).resolve().parents[2] +REPOSITORY_PRIMARY = signing.PRIMARY +REPOSITORY_SIGNING = signing.SIGNING + @unittest.skipUnless(shutil.which("gpg") and shutil.which("git"), "GnuPG and Git required") class SigningTests(unittest.TestCase): @classmethod def setUpClass(cls): - cls.fixture = tempfile.TemporaryDirectory(prefix="java-signing-fixture-") + cls.fixture = tempfile.TemporaryDirectory(prefix="repo-signing-fixture-") cls.directory = Path(cls.fixture.name) cls.keyring = cls.directory / "gnupg" cls.keyring.mkdir(mode=0o700) @@ -25,7 +34,7 @@ def gpg(*args): result = subprocess.run( ["gpg", "--no-options", "--homedir", str(cls.keyring), "--batch", "--pinentry-mode", "loopback", "--passphrase-fd", "0", *args], - input=cls.password + "\n", capture_output=True, text=True, check=True, + input=cls.password + "\n", capture_output=True, text=True, check=True, timeout=60, ) return result.stdout gpg("--quick-generate-key", "Release Test ", "rsa2048", "cert", "1d") @@ -44,125 +53,731 @@ def gpg(*args): @classmethod def tearDownClass(cls): subprocess.run(["gpgconf", "--homedir", str(cls.keyring), "--kill", "gpg-agent"], - capture_output=True, check=True) + capture_output=True, check=True, timeout=60) cls.fixture.cleanup() @contextmanager def repository(self, full=False, password=None): original = Path.cwd() - with tempfile.TemporaryDirectory(prefix="java-signing-test-") as temp: + with tempfile.TemporaryDirectory(prefix="repo-signing-test-") as temp: directory = Path(temp) - keyring = directory / "gnupg" - keyring.mkdir(mode=0o700) - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - env.update(GNUPGHOME=str(keyring), RUNNER_TEMP=str(directory), - GITHUB_ENV=str(directory / "github-env"), GIT_CONFIG_NOSYSTEM="1", - GIT_CONFIG_GLOBAL=str(directory / "initial-gitconfig"), - MAVEN_GPG_PASSPHRASE=self.password if password is None else password) - (directory / "initial-gitconfig").write_text("") - subprocess.run(["gpg", "--no-options", "--homedir", str(keyring), "--batch", - "--pinentry-mode", "loopback", "--passphrase-fd", "0", "--import", - str(self.full if full else self.private)], input=self.password + "\n", - capture_output=True, text=True, env=env, check=True) - try: - with patch.dict(os.environ, env, clear=True), \ - patch.multiple(signing, PRIMARY=self.primary, SIGNING=self.subkey, - PUBLIC_KEY=self.public): - repo = directory / "source" - signing.run(["git", "init", "--quiet", str(repo)]) - os.chdir(repo) + env = {key: value for key, value in os.environ.items() + if not key.startswith(("GIT_", "GITHUB_", "RELEASE_", "MAKE")) + and key not in ("MAVEN_GPG_PASSPHRASE", "PGP_PASSPHRASE")} + env.update(RUNNER_TEMP=str(directory), GITHUB_ENV=str(directory / "github-env"), + GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=str(directory / "gitconfig"), + RELEASE_GPG_PRIVATE_KEY_FILE=str(self.full if full else self.private), + RELEASE_GPG_PASSPHRASE=self.password if password is None else password) + Path(env["GIT_CONFIG_GLOBAL"]).write_text("") + with patch.dict(os.environ, env, clear=True), \ + patch.multiple(signing, PRIMARY=self.primary, SIGNING=self.subkey, + PUBLIC_KEY=self.public): + repo = directory / "source" + signing.run(["git", "init", "--quiet", str(repo)]) + os.chdir(repo) + try: signing.run(["git", "-c", "user.name=Initial", "-c", "user.email=initial@example.invalid", "-c", "commit.gpgsign=false", "commit", "--quiet", "--allow-empty", "-m", "Initial"]) yield directory - finally: - os.chdir(original) - subprocess.run(["gpgconf", "--homedir", str(keyring), "--kill", "gpg-agent"], - capture_output=True, check=True) + finally: + os.chdir(original) - def configured(self): - values = signing.configure() - os.environ.update(values) - self.assertEqual(Path(values["RELEASE_GPG_PASSPHRASE_FILE"]).stat().st_mode & 0o777, 0o600) - self.assertNotIn(self.password, Path(os.environ["GITHUB_ENV"]).read_text()) - return values + @contextmanager + def session(self): + with signing.signing_session() as env, patch.dict(os.environ, env, clear=True): + self.assertNotIn("RELEASE_GPG_PRIVATE_KEY", env) + self.assertNotIn("RELEASE_GPG_PRIVATE_KEY_FILE", env) + yield env def release(self): signing.run(["git", "commit", "--quiet", "--allow-empty", "-m", "Release"]) - # Maven gitexe creates annotated tags with -F rather than an explicit -s. - Path("message").write_text("Release 1.0\n") - signing.run(["git", "tag", "-F", "message", "1.0"]) - signing.run(["git", "commit", "--quiet", "--allow-empty", "-m", "Next development"]) + signing.run(["git", "tag", "-s", "-m", "Release 1.0", "v1.0.0"]) + + def test_repository_setup_failure_restores_working_directory(self): + original = Path.cwd() + real_run = signing.run + def fail_commit(command, env=None): + if "commit" in command: + raise signing.SigningError("fixture commit failed") + return real_run(command, env) + with patch.object(signing, "run", side_effect=fail_commit): + with self.assertRaisesRegex(signing.SigningError, "fixture commit failed"): + with self.repository(): + self.fail("Initial commit failure accepted") + self.assertEqual(Path.cwd(), original) + with self.repository(): + self.assertTrue(Path(".git").is_dir()) - def test_signed_release_and_cloned_checkout(self): + def test_signed_commit_and_tag_verify_after_session(self): with self.repository() as directory: - values = self.configured() - self.assertIn("-Dgpg.keyname=" + self.subkey + "!", values["MAVEN_OPTS"]) - self.release() - signing.verify_release("1.0") - # release:perform makes a separate clone: signing must survive that boundary. - clone = directory / "perform-clone" - signing.run(["git", "clone", "--quiet", "--local", ".", str(clone)]) - signing.run(["git", "-C", str(clone), "commit", "--quiet", "--allow-empty", - "-m", "Clone signing"]) - status = signing.run(["git", "-C", str(clone), "-c", "gpg.program=gpg", - "verify-commit", "--raw", "HEAD"], - signing.public_environment(Path(values["RELEASE_SIGNING_DIR"]))) - signing.verify_status(status) - signing.cleanup() - self.assertFalse(Path(values["RELEASE_SIGNING_DIR"]).exists()) + # Existing checkout settings must not override the session. + signing.run(["git", "config", "user.signingkey", "invalid-key"]) + config = Path(".git/config").read_bytes() + with self.session(): + self.release() + signing.verify_commit("HEAD") + signing.verify_tag("v1.0.0", signing.run(["git", "rev-parse", "HEAD"]).strip()) + self.assertEqual(Path(".git/config").read_bytes(), config) + self.assertFalse(list(directory.glob("release-signing-*"))) + self.assertFalse(Path(os.environ["GITHUB_ENV"]).exists()) def test_empty_passphrase_fails_before_configuration(self): with self.repository(password="") as directory: - with self.assertRaisesRegex(signing.SigningError, "nonempty"): - signing.configure() + with self.assertRaisesRegex(signing.SigningError, "nonempty"), self.session(): + self.fail("Empty passphrase accepted") self.assertFalse(list(directory.glob("release-signing-*"))) def test_wrong_passphrase_fails_with_fresh_agent(self): with self.repository(password="wrong-test-passphrase") as directory: - with self.assertRaises(signing.SigningError): - signing.configure() + with self.assertRaises(signing.SigningError), self.session(): + self.fail("Wrong passphrase accepted") self.assertFalse(list(directory.glob("release-signing-*"))) def test_primary_secret_rejected(self): with self.repository(full=True): - with self.assertRaisesRegex(signing.SigningError, "signing-private.asc"): - signing.configure() + with self.assertRaisesRegex(signing.SigningError, "signing-private.asc"), self.session(): + self.fail("Primary private key accepted") - def test_unsigned_release_commit_rejected(self): + def test_unprotected_export_rejected_after_native_import(self): + with self.repository() as directory: + home = directory / "unprotected" + home.mkdir(mode=0o700) + env = {**os.environ, "GNUPGHOME": str(home)} + def unprotected_gpg(*args): + return signing.gpg("--pinentry-mode", "loopback", "--passphrase", "", *args, env=env) + try: + unprotected_gpg("--quick-generate-key", "Unprotected Test ", + "rsa2048", "cert", "1d") + primary = [row.split(":")[9] for row in + unprotected_gpg("--with-colons", "--list-keys").splitlines() + if row.startswith("fpr:")][0] + unprotected_gpg("--quick-add-key", primary, "rsa2048", "sign", "1d") + subkey = [row.split(":")[9] for row in + unprotected_gpg("--with-colons", "--list-keys").splitlines() + if row.startswith("fpr:")][1] + private = unprotected_gpg("--armor", "--export-secret-subkeys", primary) + public = directory / "unprotected-public.asc" + public.write_text(unprotected_gpg("--armor", "--export", primary)) + finally: + signing.run(["gpgconf", "--homedir", str(home), "--kill", "gpg-agent"]) + del os.environ["RELEASE_GPG_PRIVATE_KEY_FILE"] + os.environ["RELEASE_GPG_PRIVATE_KEY"] = private + # Any nonempty passphrase would sign this export without the post-use check. + os.environ["RELEASE_GPG_PASSPHRASE"] = "unrelated-test-passphrase" + with patch.multiple(signing, PRIMARY=primary, SIGNING=subkey, PUBLIC_KEY=public), \ + self.assertRaisesRegex(signing.SigningError, "protected subkey"), self.session(): + self.fail("Unprotected subkey export accepted") + self.assertFalse(list(directory.glob("release-signing-*"))) + + def test_unsigned_commit_and_tag_rejected(self): with self.repository(): - self.configured() - self.release() - signing.run(["git", "-c", "commit.gpgsign=false", "commit", "--quiet", - "--allow-empty", "-m", "Unsigned"]) + with self.session(): + signing.run(["git", "-c", "commit.gpgsign=false", "commit", "--quiet", + "--allow-empty", "-m", "Unsigned"]) + signing.run(["git", "-c", "tag.gpgSign=false", "tag", "-a", "-m", + "Unsigned", "v1.0.0"]) + with self.assertRaises(signing.SigningError): + signing.verify_commit("HEAD") with self.assertRaises(signing.SigningError): - signing.verify_release("1.0") + signing.verify_tag("v1.0.0") - def test_unsigned_release_tag_rejected(self): + def test_tag_at_unexpected_commit_rejected(self): with self.repository(): - self.configured() - self.release() - signing.run(["git", "tag", "-d", "1.0"]) - signing.run(["git", "-c", "tag.gpgSign=false", "tag", "-a", "-m", "Unsigned", - "1.0", "HEAD~1"]) + with self.session(): + self.release() + signing.run(["git", "commit", "--quiet", "--allow-empty", "-m", "Later"]) + with self.assertRaisesRegex(signing.SigningError, "unexpected commit"): + signing.verify_tag("v1.0.0", signing.run(["git", "rev-parse", "HEAD"]).strip()) + + def test_batch_signing_reuse_and_tamper_rejection(self): + with self.repository(), self.session(): + files = [Path("package.jar"), Path("package.pom"), Path("SHA256SUMS")] + for artifact in files: + artifact.write_bytes(b"release package") + signing.sign_files(files) + signatures = [Path(str(artifact) + ".asc").read_bytes() for artifact in files] + signing.sign_files(files) + self.assertEqual([Path(str(artifact) + ".asc").read_bytes() for artifact in files], signatures) + files[0].write_bytes(b"changed package") with self.assertRaises(signing.SigningError): - signing.verify_release("1.0") + signing.sign_files(files) - def test_tag_at_unrelated_commit_rejected(self): - with self.repository(): - self.configured() - signing.run(["git", "tag", "-m", "Old commit", "1.0"]) - signing.run(["git", "commit", "--quiet", "--allow-empty", "-m", "Release"]) - with self.assertRaisesRegex(signing.SigningError, "newly signed"): - signing.verify_release("1.0") + def test_direct_sign_files_imports_credentials_and_preserves_verification_only_use(self): + with self.make_repository() as directory: + default_home = directory / "empty-default-keyring" + default_home.mkdir(mode=0o700) + os.environ["GNUPGHOME"] = str(default_home) + files = [Path("package.jar"), Path("package.pom")] + for artifact in files: + artifact.write_bytes(b"release package") + command = [sys.executable, "scripts/release/signing.py", "sign-files"] + def invoke(*artifacts): + result = subprocess.run(command + list(map(str, artifacts)), capture_output=True, + text=True, timeout=60) + self.assertNotIn(self.password, result.stdout + result.stderr) + self.assertNotIn("PRIVATE KEY", result.stdout + result.stderr) + return result + result = invoke(files[0]) + self.assertEqual(result.returncode, 0, result.stderr) + first_signature = Path(str(files[0]) + ".asc").read_bytes() + # Inline export input also opens a session, preserving existing signatures. + del os.environ["RELEASE_GPG_PRIVATE_KEY_FILE"] + os.environ["RELEASE_GPG_PRIVATE_KEY"] = self.private.read_text() + result = invoke(*files) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(str(files[0]) + ".asc").read_bytes(), first_signature) + signatures = [Path(str(artifact) + ".asc").read_bytes() for artifact in files] + del os.environ["RELEASE_GPG_PRIVATE_KEY"] + del os.environ["RELEASE_GPG_PASSPHRASE"] + result = invoke(*files) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([Path(str(artifact) + ".asc").read_bytes() for artifact in files], signatures) + files[0].write_bytes(b"tampered release") + self.assertNotEqual(invoke(*files).returncode, 0) + self.assertFalse(list(default_home.iterdir())) + self.assertFalse(list(directory.glob("release-signing-*"))) + + @unittest.skipUnless(Path("/proc/self/stat").exists(), "Linux process observation required") + def test_exec_termination_stops_child_group_before_keyring_and_agent_cleanup(self): + self.check_exec_termination("active") + + @unittest.skipUnless(Path("/proc/self/stat").exists(), "Linux process observation required") + def test_exec_startup_interruption_keeps_child_owned_until_cleanup(self): + self.check_exec_termination("startup") + + @unittest.skipUnless(Path("/proc/self/stat").exists(), "Linux process observation required") + def test_normal_teardown_interruption_finishes_agent_and_keyring_cleanup(self): + self.check_exec_termination("teardown") + + def check_exec_termination(self, window): + def running(pid): + try: + return Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] != "Z" + except FileNotFoundError: + return False + for signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + with self.subTest(signum=signum), self.make_repository() as directory: + ready, stopped = directory / "ready", directory / "stopped" + started = directory / "started" + agent_shutdown = directory / "agent-shutdown" + descendant_ready = directory / "descendant-ready" + descendant = ( + "import os,signal,time\nfrom pathlib import Path\n" + "for s in (signal.SIGTERM,signal.SIGHUP,signal.SIGINT): signal.signal(s,signal.SIG_IGN)\n" + f"Path({str(descendant_ready)!r}).write_text(str(os.getpid()))\n" + "time.sleep(60)\n") + script = ( + "import json,os,signal,subprocess,sys,time\nfrom pathlib import Path\n" + "os.close(1)\nos.close(2)\n" + f"Path({str(started)!r}).write_text(str(os.getpid()))\n" + f"child=subprocess.Popen([sys.executable,'-c',{descendant!r}])\n" + "home=Path(os.environ['GNUPGHOME'])\n" + "agent=subprocess.run(['gpg-connect-agent','--no-autostart','--homedir',str(home)," + "'GETINFO pid','/bye'],capture_output=True,text=True,check=True).stdout\n" + "agent=int(next(line[2:] for line in agent.splitlines() if line.startswith('D ')))\n" + "def interrupted(s,frame):\n" + f" Path({str(stopped)!r}).write_text(json.dumps({{'home_exists':home.exists()," + "'private_exists':(home/'private-keys-v1.d').is_dir()}))\n" + " raise SystemExit(128+s)\n" + "for s in (signal.SIGTERM,signal.SIGHUP,signal.SIGINT): signal.signal(s,interrupted)\n" + "deadline=time.monotonic()+3\n" + f"while not Path({str(descendant_ready)!r}).exists() and time.monotonic()4.0.0test' + 'release-signing-test1.0.0' + 'pom') + # Maven must supply its environment passphrase after agent expiry. + signing.run(["gpgconf", "--homedir", os.environ["GNUPGHOME"], "--kill", "gpg-agent"]) + result = subprocess.run( + ["mvn", "-B", "-ntp", "org.apache.maven.plugins:maven-gpg-plugin:3.2.4:sign", + "-Dgpg.bestPractices=true"], capture_output=True, text=True, timeout=180) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + signatures = list(Path("target").rglob("*.asc")) + self.assertEqual(len(signatures), 1) + with signing.public_environment() as public: + signing.verify_status(signing.gpg("--status-fd", "1", "--verify", signatures[0], + "pom.xml", env=public)) + + +class DiagnosticTests(unittest.TestCase): + def test_pending_interruption_does_not_launch_command(self): + for signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + with self.subTest(signum=signum), patch.object(signing.subprocess, "Popen") as spawn: + with self.assertRaises(SystemExit) as stopped: + with signing.session_signals() as interrupted: + interrupted(signum, None) + signing.execute(["test-command"], {}) + self.assertEqual(stopped.exception.code, 128 + signum) + spawn.assert_not_called() + + def test_already_exited_child_preserves_interruption_status(self): + for signum in (signal.SIGTERM, signal.SIGHUP, signal.SIGINT): + with self.subTest(signum=signum), patch.object(signing.subprocess, "Popen") as spawn, \ + patch.object(signing.os, "killpg", side_effect=ProcessLookupError): + spawn.return_value.wait.side_effect = [SystemExit(128 + signum), 0] + with self.assertRaises(SystemExit) as interrupted: + signing.execute(["test-command"], {}) + self.assertEqual(interrupted.exception.code, 128 + signum) + + def test_parsed_output_excludes_warnings_but_raw_git_keeps_status(self): + result = subprocess.CompletedProcess([], 0, "parsed-value\n", "warning\n") + with patch.object(subprocess, "run", return_value=result): + for command in [["git", "rev-parse", "HEAD"], ["git", "rev-list", "HEAD"], + ["git", "cat-file", "-t", "HEAD"], + ["gpg", "--status-fd", "1", "--verify", "artifact.sig"]]: + with self.subTest(command=command): + self.assertEqual(signing.run(command), result.stdout) + for kind in ("commit", "tag"): + self.assertEqual(signing.run(["git", "verify-" + kind, "--raw", "HEAD"]), + result.stdout + result.stderr) + + def test_failed_commands_identify_operations_without_unsafe_values(self): + sentinel = "DO-NOT-DISCLOSE" + result = subprocess.CompletedProcess([], 17, sentinel, sentinel) + with patch.object(subprocess, "run", return_value=result): + with self.assertRaisesRegex(signing.SigningError, r"Git verify-commit failed \(exit status 17\)") as error: + signing.run(["git", "verify-commit", sentinel]) + self.assertNotIn(sentinel, str(error.exception)) + with self.assertRaisesRegex(signing.SigningError, r"GPG key import failed \(exit status 17\)") as error: + signing.credential_gpg(["--import", sentinel], payload=sentinel, + env={"RELEASE_GPG_PASSPHRASE": sentinel}) + self.assertNotIn(sentinel, str(error.exception)) + + def test_command_start_and_timeout_errors_withhold_payloads(self): + for failure, category in [(OSError("DO-NOT-DISCLOSE"), "OS error"), + (subprocess.TimeoutExpired("DO-NOT-DISCLOSE", 60, + output="DO-NOT-DISCLOSE"), "timed out")]: + with self.subTest(category=category), patch.object(subprocess, "run", side_effect=failure): + with self.assertRaises(signing.SigningError) as error: + signing.gpg("--verify", "DO-NOT-DISCLOSE") + self.assertIn("GPG signature verification", str(error.exception)) + self.assertIn(category, str(error.exception)) + self.assertNotIn("DO-NOT-DISCLOSE", str(error.exception)) + + def test_unexpected_failure_identifies_operation_and_safe_category(self): + for failure, category in [(OSError("DO-NOT-DISCLOSE"), "OS error"), + (RuntimeError("DO-NOT-DISCLOSE"), "internal error")]: + output = StringIO() + with patch("sys.argv", ["signing.py", "verify-commit", "HEAD"]), \ + patch.object(signing, "verify_commit", side_effect=failure), redirect_stderr(output): + self.assertEqual(signing.main(), 1) + self.assertIn("verify-commit failed (" + category + ")", output.getvalue()) + self.assertNotIn("DO-NOT-DISCLOSE", output.getvalue()) + + +class ReleaseUserTests(unittest.TestCase): + def test_actions_accepts_workflow_actors_without_local_gh_authentication(self): + with tempfile.TemporaryDirectory() as temp: + gh = Path(temp) / "gh" + gh.write_text("#!/bin/sh\nexit 99\n") + gh.chmod(0o700) + for actor, triggering_actor in [("release-operator", "release-operator"), + ("scylladb-publisher", "rerun-operator"), + ("release-operator", "")]: + with self.subTest(actor=actor, triggering_actor=triggering_actor): + result = subprocess.run( + ["make", "release-check-user"], cwd=ROOT, capture_output=True, text=True, timeout=60, + env={**os.environ, "PATH": temp + os.pathsep + os.environ["PATH"], + "GITHUB_ACTIONS": "true", "GITHUB_ACTOR": actor, + "GITHUB_TRIGGERING_ACTOR": triggering_actor}, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_local_release_commands_automatically_reject_other_users(self): + with tempfile.TemporaryDirectory() as temp: + gh = Path(temp) / "gh" + gh.write_text("#!/bin/sh\nprintf '%s\\n' someone-else\n") + gh.chmod(0o700) + for target in ["release-prepare", "release", "release-dry-run", "release-preflight", + "_release-prepare", "_release", "_release-dry-run"]: + with self.subTest(target=target): + result = subprocess.run( + ["make", target], cwd=ROOT, capture_output=True, text=True, timeout=60, + env={**os.environ, "PATH": temp + os.pathsep + os.environ["PATH"], + "GITHUB_ACTIONS": "false"}, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Release requires GitHub user scylladb-publisher", result.stderr) + self.assertNotIn("signing.py exec", result.stdout) + + def test_local_release_uses_authenticated_gh_user(self): + with tempfile.TemporaryDirectory() as temp: + gh = Path(temp) / "gh" + gh.write_text('#!/bin/sh\nprintf "%s\\n" "$TEST_GH_USER"\n') + gh.chmod(0o700) + env = {**os.environ, "PATH": temp + os.pathsep + os.environ["PATH"], + "GITHUB_ACTIONS": "false", "TEST_GH_USER": "someone-else"} + result = subprocess.run(["make", "release-prepare"], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + self.assertNotEqual(result.returncode, 0) + self.assertIn("current: someone-else", result.stderr) + env["TEST_GH_USER"] = "scylladb-publisher" + subprocess.run(["make", "release-check-user"], cwd=ROOT, env=env, check=True, + capture_output=True, timeout=60) + gh.write_text("#!/bin/sh\nexit 1\n") + result = subprocess.run(["make", "release-prepare"], cwd=ROOT, env=env, + capture_output=True, text=True, timeout=60) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Run gh auth login or gh auth switch", result.stderr) if __name__ == "__main__": diff --git a/scripts/release/test_target.py b/scripts/release/test_target.py new file mode 100644 index 00000000000..5b7f34573f5 --- /dev/null +++ b/scripts/release/test_target.py @@ -0,0 +1,181 @@ +"""Driver release policy and workflow tests, without production credentials.""" + +from contextlib import contextmanager +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import guard +import target + +ROOT = Path(__file__).resolve().parents[2] + + +class TargetTests(unittest.TestCase): + def git(self, directory, *args): + return subprocess.check_output([ + 'git', '-c', 'commit.gpgsign=false', '-c', 'tag.gpgSign=false', + '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-C', str(directory), *args, + ], text=True, stderr=subprocess.PIPE).strip() + + @contextmanager + def fixture(self, major): + previous = Path.cwd() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'source' + source.mkdir() + self.git(source, 'init', '-q', '--initial-branch=scylla-' + major + '.x') + commits = [] + for value in (major + '.1.2.3-SNAPSHOT', major + '.1.2.3', major + '.1.2.4-SNAPSHOT'): + (source / 'pom.xml').write_text( + '' + '' + value + '') + self.git(source, 'add', '.') + self.git(source, 'commit', '-qm', 'Version ' + value) + commits.append(self.git(source, 'rev-parse', 'HEAD')) + if value == major + '.1.2.3': + self.git(source, 'tag', value) + self.git(root, 'clone', '-q', str(source), str(root / 'checkout')) + os.chdir(root / 'checkout') + try: + yield root, source, commits + finally: + os.chdir(previous) + + def test_both_branch_versions_overrides_and_historical_matching(self): + for major in ('3', '4'): + with self.subTest(major=major), self.fixture(major) as (_, _, commits): + branch = f'scylla-{major}.x' + ref = 'refs/heads/' + branch + self.assertEqual(target.resolve(major, branch, '', commits[-1], ref), + (commits[-1], major + '.1.2.4')) + self.assertEqual(target.resolve(major, branch, major + '.9.8.7', commits[-1], ref), + (commits[-1], major + '.9.8.7')) + self.assertEqual(target.resolve(major, major + '.1.2.3', '', commits[-1], ref), + (commits[0], major + '.1.2.3')) + for invalid in ('feature', 'scylla-5.x', 'v' + major + '.1.2.3', '--help', + major + '.1.2', '../other'): + with self.subTest(invalid=invalid), self.assertRaises(target.Invalid): + target.resolve(major, invalid, '', commits[-1], ref) + for invalid in ('2.1.2.3', '3.1.2' if major == '3' else '4.1.2', + major + '.1.2.3-SNAPSHOT', major + '.1.2.3\n'): + with self.assertRaises(target.Invalid): + target.resolve(major, branch, invalid, commits[-1], ref) + with self.assertRaises(target.Invalid): + target.resolve(major, branch, '', commits[-1], 'refs/heads/feature') + with self.assertRaises(target.Invalid): + target.resolve(major, major + '.1.2.3', major + '.9.8.7', commits[-1], ref) + self.git(Path.cwd(), 'tag', major + '.1.2.5', commits[-1]) + with self.assertRaises(target.Invalid): + target.resolve(major, major + '.1.2.5', '', commits[-1], ref) + + def test_branch_moving_during_approval_is_rejected_for_both_lines(self): + for major in ('3', '4'): + with self.subTest(major=major), self.fixture(major) as (_, source, commits): + target.checkout(major, f'scylla-{major}.x', commits[-1]) + self.assertEqual(target.git('rev-parse', 'HEAD'), commits[-1]) + self.git(source, 'commit', '--allow-empty', '-qm', 'Advance') + with self.assertRaisesRegex(target.Invalid, 'advanced'): + target.checkout(major, f'scylla-{major}.x', commits[-1]) + + def test_historical_checkout_uses_validated_sha_even_when_tag_moves(self): + for major in ('3', '4'): + with self.subTest(major=major), self.fixture(major) as (_, source, commits): + self.git(source, 'tag', '-f', major + '.1.2.3', commits[-1]) + target.checkout(major, major + '.1.2.3', commits[0]) + self.assertEqual(target.git('rev-parse', 'HEAD'), commits[0]) + self.assertEqual(target.git('tag', '--list', major + '.1.2.3'), '') + + def test_explicit_push_from_detached_checkout_has_no_upstream_dependency(self): + workflow = (ROOT / '.github/workflows/release.yml').read_text() + push = next(script for script in guard.workflow_commands(workflow) if 'git push origin' in script) + major = '3' if 'HEAD:refs/heads/scylla-3.x' in push else '4' + with self.fixture(major) as (root, source, _): + bare = root / 'remote.git' + self.git(root, 'clone', '-q', '--bare', str(source), str(bare)) + self.git(Path.cwd(), 'remote', 'set-url', 'origin', str(bare)) + self.git(Path.cwd(), 'checkout', '--detach') + self.git(Path.cwd(), 'commit', '--allow-empty', '-qm', 'Release') + self.git(Path.cwd(), 'tag', '-a', 'release-fixture', '-m', 'Release') + prepared = target.git('rev-parse', 'HEAD') + result = subprocess.run(['bash', '-eo', 'pipefail', '-c', push], capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.git(bare, 'rev-parse', f'refs/heads/scylla-{major}.x'), prepared) + self.assertEqual(self.git(bare, 'rev-parse', 'refs/tags/release-fixture^{}'), prepared) + + def test_placeholder_or_empty_central_credentials_fail_before_preparation(self): + for name in ('SONATYPE_TOKEN_USERNAME', 'SONATYPE_TOKEN_PASSWORD'): + for value in ('', 'REPLACE_ME', ' REPLACE_ME ', ' \n'): + with self.subTest(name=name, value=value), patch.dict(os.environ, { + 'SONATYPE_TOKEN_USERNAME': 'synthetic-user', + 'SONATYPE_TOKEN_PASSWORD': 'synthetic-password', name: value}, clear=True): + with self.assertRaisesRegex(target.Invalid, name): + target.preflight() + + def test_blocker_scope_and_pagination_for_both_lines(self): + for major in ('3', '4'): + other = '4' if major == '3' else '3' + for lines, blocks in (([], True), ([major], True), ([other], False), + ([major, other], True)): + issue = {'html_url': 'https://example.invalid/issue', + 'labels': [{'name': f'area/Driver_-_java-driver-{line}.x'} for line in lines]} + payload = json.dumps([[], [issue]]) + with patch.dict(os.environ, {'GITHUB_REPOSITORY': 'test/repo'}), \ + patch.object(subprocess, 'check_output', return_value=payload) as query: + if blocks: + with self.assertRaises(target.Invalid): + target.blockers(major) + else: + target.blockers(major) + self.assertIn('--paginate', query.call_args.args[0]) + + def test_readme_stamps_effective_version(self): + previous = Path.cwd() + with tempfile.TemporaryDirectory() as directory: + os.chdir(directory) + try: + Path('README.md').write_text('Use 3.11.5.20 and 3.11.5.20, with Java 8.\n') + with patch.dict(os.environ, {'RELEASE_VERSION': '3.11.6.1'}), patch.object(target, 'git'): + target.stamp_readme() + self.assertEqual(Path('README.md').read_text(), + 'Use 3.11.6.1 and 3.11.6.1, with Java 8.\n') + finally: + os.chdir(previous) + + def test_approval_is_after_validation_and_matrix_and_has_no_shared_cache(self): + workflow = (ROOT / '.github/workflows/release.yml').read_text() + validate, publish = workflow.split('\n release:\n', 1) + self.assertIn('needs: [validate, pre-release-integration]', publish) + self.assertIn("needs.validate.result == 'success'", publish) + self.assertIn("inputs.skip-integration-matrix && needs.pre-release-integration.result == 'skipped'", publish) + self.assertNotIn('secrets.', validate) + self.assertNotIn('environment:', validate) + self.assertNotIn('contents: write', validate) + self.assertIn('python3 -m unittest discover', validate) + self.assertIn('driver_ref: ${{ needs.validate.outputs.ref }}', validate) + self.assertIn('driver_version: ${{ needs.validate.outputs.version }}', validate) + self.assertIn(' environment: release\n', publish) + self.assertNotIn('cache:', publish) + self.assertNotIn('download-artifact', publish) + self.assertIn('ref: ${{ github.sha }}', publish) + self.assertEqual(publish.count("RELEASE_SKIP_TESTS: 'true'"), 3) + self.assertLess(publish.index('Preserve release guard'), publish.index('Checkout release target')) + self.assertLess(publish.index('Check Central credentials'), publish.index('Prepare release')) + self.assertLess(publish.index('Check release credential safety'), publish.index('secrets.')) + self.assertNotIn('gpg-private-key:', publish) + self.assertNotIn('signing.py" configure', publish) + self.assertIn('verify-prepared', publish) + self.assertLess(publish.index('Recheck for open release blockers'), publish.index('Perform release')) + self.assertIn('inputs.dry-run == false', publish.split('Push changes to SCM')[1]) + self.assertIn("inputs.dry-run == false", publish.split('publish-docs:')[1]) + + +if __name__ == '__main__': + unittest.main() From cccb737d1859615efed933f89fea35c24a0fbdd6 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Fri, 23 Oct 2026 12:44:53 -0400 Subject: [PATCH 2/2] Clarify the disposable passphrase fixture for security analysis Write the literal test-only passphrase into an owner-only file so static analysis can distinguish the fixture from production credential storage. --- scripts/release/test_signing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/release/test_signing.py b/scripts/release/test_signing.py index 580f8eedb1d..1c821e0ff3b 100644 --- a/scripts/release/test_signing.py +++ b/scripts/release/test_signing.py @@ -395,7 +395,8 @@ def test_exec_cleans_up_after_command_failure(self): def test_exec_accepts_passphrase_file(self): with self.repository() as directory: password = directory / "passphrase" - password.write_text(self.password + "\n") + # Literal fixture data belongs only to the disposable test key. + signing.write_private(password, "test-only-release-passphrase\n") os.environ["RELEASE_GPG_PASSPHRASE_FILE"] = str(password) del os.environ["RELEASE_GPG_PASSPHRASE"] with patch("sys.argv", ["signing.py", "exec", "git", "commit", "--quiet",