diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 409842b..46c1365 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,8 @@ jobs: run: node --test scripts/ci/__tests__/release-policy.test.mjs - name: Test public release input wiring run: node --test scripts/ci/__tests__/public-release-inputs.test.mjs + - name: Test public-to-private merge safety + run: node --test scripts/ci/__tests__/sync-public-to-private.test.mjs # Every surface that can write a verbatim memory must be able to stamp # content_class. The 0.2.0 release fixed Hermes for Core's raw-content # policy and missed OpenClaw, whose published plugin could not perform a diff --git a/.github/workflows/publish-core-docker.yml b/.github/workflows/publish-core-docker.yml index f8dd930..62aebb8 100644 --- a/.github/workflows/publish-core-docker.yml +++ b/.github/workflows/publish-core-docker.yml @@ -36,6 +36,11 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 steps: + - name: Install artifact scanner dependency + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends ripgrep + rg --version - name: Setup Node uses: actions/setup-node@v4 with: diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 9740e96..3a85715 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -102,6 +102,11 @@ jobs: MANIFEST_JSON: ${{ needs.manifest.outputs.manifest_json }} PUBLIC_SHA: ${{ needs.manifest.outputs.public_sha }} steps: + - name: Install artifact scanner dependency + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends ripgrep + rg --version - name: Checkout public_sha uses: actions/checkout@v4 with: diff --git a/.github/workflows/sync-to-private.yml b/.github/workflows/sync-to-private.yml index 6b95748..e9315da 100644 --- a/.github/workflows/sync-to-private.yml +++ b/.github/workflows/sync-to-private.yml @@ -7,11 +7,8 @@ on: jobs: sync: - # This realign (force-push public main -> atomicmemory-internal main) must - # run ONLY from the public repo. The workflow file is mirrored into - # atomicmemory-internal by the public export, so without this guard it also - # fires on internal merges, where SYNC_TOKEN is absent and the push fails - # (and a private->private realign would be meaningless anyway). + # Merge public updates without discarding private-only files or newer work. + # This workflow is also exported to the private repository; never run there. if: github.repository == 'atomicstrata/atomicmemory' runs-on: ubuntu-latest steps: @@ -32,5 +29,4 @@ jobs: # Add the atomicmemory-internal repository as a secure remote git remote add private-target "https://x-access-token:${SYNC_TOKEN}@github.com/atomicstrata/atomicmemory-internal.git" - # Force push the main branch to the private repo to ensure perfect alignment - git push private-target HEAD:main --force + bash scripts/ci/sync-public-to-private.sh private-target "$GITHUB_SHA" diff --git a/scripts/ci/__tests__/public-release-inputs.test.mjs b/scripts/ci/__tests__/public-release-inputs.test.mjs index 055f308..a59ac76 100644 --- a/scripts/ci/__tests__/public-release-inputs.test.mjs +++ b/scripts/ci/__tests__/public-release-inputs.test.mjs @@ -29,6 +29,23 @@ for (const [name, steps, prefix] of [ ['docker', docker.jobs.publish.steps, 'release-source/'], ]) { const step = steps.find((s) => s.name.includes('private identifiers')); + test(`${name}: installs ripgrep before scanning artifacts`, () => { + const setup = steps.find((s) => s.name === 'Install artifact scanner dependency'); + assert.ok(setup); + assert.ok(steps.indexOf(setup) < steps.indexOf(step)); + assert.equal(setup.if, undefined); + const commands = []; + const result = spawnSync('bash', ['-e', '-c', + 'sudo() { printf "%s\\n" "$*"; }; rg() { printf "rg %s\\n" "$*"; };\n' + setup.run], + { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + commands.push(...result.stdout.trim().split('\n')); + assert.deepEqual(commands, [ + 'apt-get update', + 'apt-get install --yes --no-install-recommends ripgrep', + 'rg --version', + ]); + }); test(`${name}: secret reaches scanner, missing secret refuses release`, () => { assert.equal(step.env.PUBLIC_ARTIFACT_SIGNATURES, secretExpression); const dir = mkdtempSync(join(tmpdir(), 'am-release-input-')); diff --git a/scripts/ci/__tests__/sync-public-to-private.test.mjs b/scripts/ci/__tests__/sync-public-to-private.test.mjs new file mode 100644 index 0000000..a1cbec0 --- /dev/null +++ b/scripts/ci/__tests__/sync-public-to-private.test.mjs @@ -0,0 +1,95 @@ +/** Exercise the production sync against real local Git remotes. */ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnSync } from 'node:child_process'; +import { test } from 'node:test'; + +const script = fileURLToPath(new URL('../sync-public-to-private.sh', import.meta.url)); +const env = { ...process.env, GIT_AUTHOR_NAME: 'Fixture', GIT_AUTHOR_EMAIL: 'fixture@example.com', GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.com', GIT_TERMINAL_PROMPT: '0' }; +function git(cwd, ...args) { + const result = spawnSync('git', args, { cwd, env, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return result.stdout.trim(); +} +function commit(dir, file, value) { + mkdirSync(join(dir, file, '..'), { recursive: true }); + writeFileSync(join(dir, file), value); + git(dir, 'add', file); + git(dir, 'commit', '-m', `Update ${file}`); + return git(dir, 'rev-parse', 'HEAD'); +} +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'am-sync-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const source = join(root, 'source'); + const remote = join(root, 'private.git'); + const runner = join(root, 'runner'); + git(root, 'init', '-b', 'main', source); + commit(source, 'shared.txt', 'baseline\n'); + commit(source, '.gitattributes', 'packages/core/hosted export-ignore\n'); + git(root, 'clone', '--bare', source, remote); + git(root, 'clone', source, runner); + git(runner, 'remote', 'add', 'private-target', remote); + commit(source, 'packages/core/hosted/overlay.mjs', 'private fixture\n'); + git(source, 'push', remote, 'HEAD:main'); + return { root, source, remote, runner }; +} +function sync(f) { + return spawnSync('bash', [script, 'private-target', git(f.runner, 'rev-parse', 'HEAD')], { cwd: f.runner, env, encoding: 'utf8' }); +} + +test('merges public updates and keeps private files/history out of the export', (t) => { + const f = fixture(t); + commit(f.source, 'internal.txt', 'new private work\n'); + git(f.source, 'push', f.remote, 'HEAD:main'); + const previous = git(f.source, 'rev-parse', 'HEAD'); + const published = commit(f.runner, 'shared.txt', 'public update\n'); + const result = sync(f); + assert.equal(result.status, 0, result.stderr); + assert.equal(git(f.root, '--git-dir', f.remote, 'show', 'main:packages/core/hosted/overlay.mjs'), 'private fixture'); + assert.equal(git(f.root, '--git-dir', f.remote, 'show', 'main:shared.txt'), 'public update'); + assert.equal(git(f.root, '--git-dir', f.remote, 'show', 'main:internal.txt'), 'new private work'); + git(f.runner, 'merge-base', '--is-ancestor', previous, 'HEAD'); + git(f.runner, 'merge-base', '--is-ancestor', published, 'HEAD'); + const archive = spawnSync('git', ['archive', 'HEAD'], { cwd: f.runner }); + const paths = spawnSync('tar', ['-tf', '-'], { input: archive.stdout, encoding: 'utf8' }); + assert.equal(archive.status, 0); + assert.equal(paths.status, 0); + assert.match(paths.stdout, /shared.txt/); + assert.doesNotMatch(paths.stdout, /packages\/core\/hosted/); + assert.equal(sync(f).status, 0, 'rerun is idempotent'); +}); + +test('conflicting shared edits leave private main untouched', (t) => { + const f = fixture(t); + const before = commit(f.source, 'shared.txt', 'private edit\n'); + git(f.source, 'push', f.remote, 'HEAD:main'); + commit(f.runner, 'shared.txt', 'public edit\n'); + const result = sync(f); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /merge conflicted/); + assert.equal(git(f.root, '--git-dir', f.remote, 'rev-parse', 'main'), before); + assert.equal(git(f.runner, 'status', '--porcelain'), ''); +}); + +test('a concurrent private push is rejected rather than overwritten', (t) => { + const f = fixture(t); + const newer = commit(f.source, 'concurrent.txt', 'must survive\n'); + git(f.source, 'push', f.remote, 'HEAD:refs/heads/race'); + commit(f.runner, 'public.txt', 'public change\n'); + const hook = join(f.runner, '.git/hooks/pre-push'); + writeFileSync(hook, `#!/bin/sh\ngit --git-dir='${f.remote}' update-ref refs/heads/main ${newer}\n`, { mode: 0o755 }); + const result = sync(f); + assert.notEqual(result.status, 0); + assert.equal(git(f.root, '--git-dir', f.remote, 'rev-parse', 'main'), newer); + assert.equal(git(f.root, '--git-dir', f.remote, 'show', 'main:concurrent.txt'), 'must survive'); +}); + +test('workflow calls the tested merge path, never a force-push', () => { + const workflow = readFileSync(new URL('../../../.github/workflows/sync-to-private.yml', import.meta.url), 'utf8'); + assert.match(workflow, /bash scripts\/ci\/sync-public-to-private\.sh private-target "\$GITHUB_SHA"/); + assert.doesNotMatch(workflow, /--force/); +}); diff --git a/scripts/ci/sync-public-to-private.sh b/scripts/ci/sync-public-to-private.sh new file mode 100644 index 0000000..7d54167 --- /dev/null +++ b/scripts/ci/sync-public-to-private.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Merge a public snapshot into private main, preserving private-only history/files. +set -euo pipefail + +remote="${1:?private remote is required}" +public_commit="${2:?public commit is required}" + +if [[ -n "$(git status --porcelain)" ]]; then + echo 'Public-to-private sync requires a clean checkout.' >&2 + exit 1 +fi + +git rev-parse --verify "${public_commit}^{commit}" >/dev/null +git fetch --no-tags "$remote" refs/heads/main +git switch --detach FETCH_HEAD + +# A conflict must be resolved internally, never by replacing the private tree. +if ! git merge --no-edit "$public_commit"; then + git merge --abort + echo 'Public-to-private merge conflicted; resolve it in the private repository.' >&2 + exit 1 +fi + +# Ordinary push rejects a concurrent private update. Never force or auto-retry. +git push "$remote" HEAD:refs/heads/main