Generate recipes and build workflows with vinca-rs

.github/actions/build-recipes/action.yaml

@@ -0,0 +1,76 @@
+# Build and upload some of a distribution's recipes: the steps of every job of the
+# build workflows (.github/build-template.yaml). One action keeps each generated job
+# a few lines long, so many more jobs fit GitHub's 500 KiB workflow file limit.
+name: Build recipes
+description: Build the given recipes of distros/<distro>/work/recipes and upload them
+inputs:
+  recipes:
+    description: Space-separated recipe names
+    required: true
+  platform:
+    description: Target platform (conda subdir)
+    required: true
+  token:
+    description: anaconda.org API token of the distribution's organization
+    required: false
+  keep-file:
+    description: File kept as an artifact (the build times the job recorded)
+    required: false
+runs:
+  using: composite
+  steps:
+    # git auto-maintenance can leave a lock file in the git clones pixi caches,
+    # which breaks every later run restoring that cache
+    - name: Disable git auto-maintenance
+      shell: bash
+      run: git config --global maintenance.auto false && git config --global gc.auto 0
+    - name: Setup pixi
+      uses: prefix-dev/setup-pixi@v0.11.0
+      with:
+        pixi-version: v0.81.0
+        cache: "true"
+        log-level: v
+        frozen: "true"
+    - if: runner.os == 'Windows'
+      uses: egor-tensin/cleanup-path@v5
+      with:
+        dirs: C:\Program Files\Git\usr\bin;C:\Program Files\Git\bin;C:\Program Files\Git\cmd;C:\Program Files\Git\mingw64\bin
+    - if: runner.os == 'Windows'
+      name: conda-forge build setup
+      shell: cmd
+      run: |
+        :: hosted runners may have libcrypto/libssl DLLs in System32, found before
+        :: the environment's openssl
+        DEL C:\Windows\System32\libcrypto-1_1-x64.dll || (Echo Ignoring failure to delete libcrypto)
+        DEL C:\Windows\System32\libssl-1_1-x64.dll || (Echo Ignoring failure to delete libssl)
+    - if: runner.os == 'Windows'
+      name: Build
+      shell: cmd
+      env:
+        ANACONDA_API_TOKEN: ${{ inputs.token }}
+        CURRENT_RECIPES: ${{ inputs.recipes }}
+        CONDA_BLD_PATH: C:\\bld\\
+        VINCA_CUSTOM_CMAKE_BUILD_DIR: C:\\x\\
+        PYTHONUNBUFFERED: "1"
+      run: call .scripts\build_win.bat
+    - if: runner.os != 'Windows'
+      name: Build
+      shell: bash
+      env:
+        ANACONDA_API_TOKEN: ${{ inputs.token }}
+        CURRENT_RECIPES: ${{ inputs.recipes }}
+        BUILD_TARGET: ${{ inputs.platform }}
+        PYTHONUNBUFFERED: "1"
+        CI: github
+      run: |
+        export GIT_BRANCH=${GITHUB_REF_NAME}
+        export FEEDSTOCK_NAME=${GITHUB_REPOSITORY##*/}
+        .scripts/build_unix.sh --target "$BUILD_TARGET"
+    - name: Keep build times
+      if: always() && inputs.keep-file != ''
+      uses: actions/upload-artifact@v6
+      with:
+        name: ${{ github.job }}-${{ github.run_attempt }}
+        path: ${{ inputs.keep-file }}
+        if-no-files-found: ignore
+        retention-days: 30

.github/build-template.yaml

@@ -0,0 +1,48 @@
+# Template of the build workflows (`pixi run rs <distro> gha`, vinca-rs workflow).
+# Not a workflow itself: vinca-rs copies jobs.build once per job, sets each copy's
+# name and needs (exactly the jobs building its recipes' dependencies) and replaces
+# @recipes@ with its recipes; @distro@, @platform@, @branch@, @secret@ and
+# @keep_file@ come from tools/robostack.py. Selectors (if/then/else) are resolved
+# for the platform. The steps are .github/actions/build-recipes.
+name: Build branch
+run-name: "@distro@ @platform@"
+on:
+  push:
+    branches: ["@branch@"]
+# one run per distribution and platform at a time: a newer run (the latest main's
+# recipes and workflow) cancels the older one, and skips what that one uploaded
+# already. Re-run failed jobs only in the newest run.
+concurrency:
+  group: build-@distro@-@platform@
+  cancel-in-progress: true
+env:
+  ROBOSTACK_DISTRO: "@distro@"
+  # optional repository variable: upload to this prefix.dev channel instead
+  # (e.g. a test channel), or 'none' to only build
+  ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }}
+jobs:
+  build:
+    runs-on:
+      if: linux and x86_64
+      then: ubuntu-latest
+      else:
+        if: linux
+        then: ubuntu-24.04-arm
+        else:
+          if: osx and x86_64
+          then: macos-15-intel
+          else:
+            if: osx
+            then: macos-15
+            else: windows-2022
+    # jobs that declare permissions get none they don't list; prefix.dev uploads use
+    # trusted publishing (id-token), attestations are published too
+    permissions: {contents: read, id-token: write, attestations: write}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build-recipes
+        with:
+          recipes: "@recipes@"
+          platform: "@platform@"
+          token: ${{ secrets.@secret@ }}
+          keep-file: "@keep_file@"

.github/workflows/testpr.yaml

@@ -1,6 +1,6 @@
 # Pull-request build: only the distributions a PR touches are built
 # (a change outside distros/ affects all of them), on all five platforms.
-name: Build
+name: Pull request
 
 on:
   pull_request:
@@ -10,8 +10,9 @@ on:
         description: Space-separated distributions to build (default all)
         required: false
 
+# a fixed key, not github.workflow: renaming the workflow mustn't let two runs overlap
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  group: pr-build-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 permissions:
@@ -24,6 +25,7 @@ env:
 
 jobs:
   plan:
+    name: "plan: distros and platforms to build"
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -67,6 +69,7 @@ jobs:
           echo "targets=$(pixi run python tools/robostack.py build-targets --labels "$labels")" | tee -a "$GITHUB_OUTPUT"
 
   check:
+    name: "checks: tooling tests, generated files"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
@@ -97,11 +100,11 @@ jobs:
       fail-fast: false
       matrix:
         distro: ${{ fromJSON(needs.plan.outputs.distros) }}
-    name: ${{ matrix.distro }} patches
+    name: "${{ matrix.distro }}: patches apply"
     runs-on: ubuntu-latest
     env:
       DISTRO: ${{ matrix.distro }}
-      # generate-recipes -n leaves out what is on the channel, like the builds do
+      # generate-recipes leaves out what is on the channel (same build string), like the builds do
       ROBOSTACK_UPLOAD_CHANNEL: ${{ vars.ROBOSTACK_UPLOAD_CHANNEL }}
     steps:
       - uses: actions/checkout@v6
@@ -123,7 +126,7 @@ jobs:
           all=$(mktemp -d)
           for platform in linux-64 linux-aarch64 osx-64 osx-arm64 win-64; do
             for attempt in 1 2 3; do
-              if pixi run rs "$DISTRO" generate-recipes --platform "$platform" -n; then break; fi
+              if pixi run rs "$DISTRO" generate-recipes --platform "$platform"; then break; fi
               if [[ "$attempt" == 3 ]]; then echo "Recipe generation failed for $platform" >&2; exit 1; fi
               sleep 15
             done
@@ -147,7 +150,7 @@ jobs:
       matrix:
         distro: ${{ fromJSON(needs.plan.outputs.distros) }}
         target: ${{ fromJSON(needs.plan.outputs.targets) }}  # tools/robostack.py BUILD_TARGETS
-    name: ${{ matrix.distro }} ${{ matrix.target.platform }}
+    name: "${{ matrix.distro }} ${{ matrix.target.platform }}: build"
     runs-on: ${{ matrix.target.os }}
     env:
       DISTRO: ${{ matrix.distro }}
@@ -217,7 +220,7 @@ jobs:
       run: |
         set -e
         for attempt in 1 2 3; do
-          if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM" -n; then break; fi
+          if pixi run -v rs "$DISTRO" generate-recipes --platform "$PLATFORM"; then break; fi
           if [[ "${attempt}" == "3" ]]; then echo "Recipe generation failed after 3 attempts" >&2; exit 1; fi
           echo "Recipe generation attempt ${attempt} failed; retrying..." >&2
           sleep 15

.scripts/build_unix.sh

@@ -40,14 +40,16 @@ pixi run -v rs "${ROBOSTACK_DISTRO}" build-ci "${recipe_args[@]}" --output-dir $
 
 # Check if it build something, this is a hotfix for the skips inside additional_recipes
 shopt -s nullglob
-conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda )
+# noarch packages (the ros-<distro>-* compatibility packages) are built and uploaded
+# by every platform's jobs: the first upload wins, the others are skipped
+conda_packages=( "${CONDA_BLD_PATH}/${target}"*/*.conda "${CONDA_BLD_PATH}/noarch/"*.conda )
 if (( ${#conda_packages[@]} > 0 )); then
     # Upload packages one-by-one; the upload task (tools/robostack.py) uploads them to
     # prefix.dev and anaconda.org and skips packages that are already there.
     for conda_package in "${conda_packages[@]}"; do
         pixi run rs "${ROBOSTACK_DISTRO}" upload "${conda_package}"
     done
 else
-    echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target}"
+    echo "Warning: No .conda files found in ${CONDA_BLD_PATH}/${target} or ${CONDA_BLD_PATH}/noarch"
     echo "This might be due to all the packages being skipped"
 fi

.scripts/build_win.bat

@@ -37,17 +37,22 @@ for %%X in (%CURRENT_RECIPES%) do set "RECIPE_ARGS=!RECIPE_ARGS! --recipe %FEEDS
 pixi run -v rs %ROBOSTACK_DISTRO% build-ci !RECIPE_ARGS! --output-dir %CONDA_BLD_PATH%
 if errorlevel 1 exit 1
 
-:: Check if .conda files exist in the win-64 directory
-if exist "%CONDA_BLD_PATH%\win-64\*.conda" (
-    echo Found .conda files, starting upload...
-    rem Upload packages one-by-one; the upload task (tools/robostack.py) uploads them to
-    rem prefix.dev and anaconda.org and skips packages that are already there.
-    for %%F in ("%CONDA_BLD_PATH%\win-64\*.conda") do (
-        echo Uploading %%~fF
-        pixi run rs %ROBOSTACK_DISTRO% upload "%%~fF"
-        if errorlevel 1 exit 1
+:: Upload the win-64 and noarch packages one-by-one; the upload task (tools/robostack.py)
+:: uploads them to prefix.dev and anaconda.org and skips packages that are already there
+:: (noarch ones, the ros-<distro>-* compatibility packages, are built and uploaded by
+:: every platform's jobs: the first upload wins).
+set "UPLOADED="
+for %%D in (win-64 noarch) do (
+    if exist "%CONDA_BLD_PATH%\%%D\*.conda" (
+        for %%F in ("%CONDA_BLD_PATH%\%%D\*.conda") do (
+            echo Uploading %%~fF
+            pixi run rs %ROBOSTACK_DISTRO% upload "%%~fF"
+            if errorlevel 1 exit 1
+            set "UPLOADED=1"
+        )
     )
-) else (
-    echo Warning: No .conda files found in %CONDA_BLD_PATH%\win-64
+)
+if not defined UPLOADED (
+    echo Warning: No .conda files found in %CONDA_BLD_PATH%\win-64 or %CONDA_BLD_PATH%\noarch
     echo This might be due to all the packages being skipped
 )

README.md

@@ -16,7 +16,8 @@ distribution comes from the channel. The older names keep working: for every pac
 there is also a small `ros-<distro>-<package>` package that just depends on
 `ros2-<package>` of the same version (vinca's `package_name_mode: both` in
 `shared/vinca.yaml`), so existing environments and instructions such as
-`pixi add ros-jazzy-desktop` don't break.
+`pixi add ros-jazzy-desktop` don't break. These are `noarch` packages, built once for
+every platform.
 
 **A package is missing?** Open a *Package request* issue. robostack-bot opens a pull
 request that adds it to the build, and a maintainer reviews it.

distros/humble/pkg_additional_info.yaml

@@ -115,6 +115,8 @@ rtabmap_slam:  # its recipe test stops the node for sure (package tests hung on
   build_number: 23
 spatio_temporal_voxel_layer:
   build_number: 23
+tracetools:
+  build_number: 24
 urdf:  # drop deprecated urdf_world/types.h include (ros2/urdf#54) for headers 3.x
   build_number: 23
 urdf_parser_plugin:  # use urdf_model/types.h (ros2/urdf#54) for headers 3.x

distros/humble/rosdistro_additional_recipes.yaml

@@ -82,13 +82,6 @@ med7_moveit_config:
   rev: 2e932d4e062966afe216710a7cc2fe50a8099365
   version: 0.3.0
   additional_folder: lbr_moveit_config/med7_moveit_config
-mujoco_vendor:
-  # jazzy's 0.1.0: unlike humble's 0.0.8 it honours AMENT_VENDOR_POLICY, so it uses
-  # conda-forge's libmujoco (shared pkg_additional_info) instead of downloading MuJoCo
-  # and symlinking it (rejected on Windows; macOS isn't supported by the download)
-  tag: release/jazzy/mujoco_vendor/0.1.0-1
-  url: https://github.com/ros2-gbp/mujoco_vendor-release.git
-  version: 0.1.0
 opennav_coverage:
   # Not released in rosdistro; humble-v2 branch (Fields2Cover v2)
   url: https://github.com/open-navigation/opennav_coverage.git

distros/jazzy/rosdistro_additional_recipes.yaml

@@ -145,7 +145,7 @@ ros2_easy_test:
   version: 0.3.0
   additional_folder: ros2_easy_test
 rplidar_ros:
-  # 2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only
+  override: "2.1.4 (as humble) for its win32 SDK arch; the released 2.1.0 is POSIX only"
   tag: 2.1.4
   url: https://github.com/Slamtec/rplidar_ros.git
   version: 2.1.4