feat: introduce recipes for development environments

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Test
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: jiro4989/setup-nim-action@v2
+        with:
+          nim-version: "2.2.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install dependencies
+        run: make dev-setup
+      - name: Build and run isolated tests
+        run: make test
+      - name: Validate public examples
+        run: |
+          for recipe in examples/recipes/*.yaml; do
+            bin/ocdev recipe validate "$recipe" --json
+          done
+          for project in examples/projects/*/project.yaml; do
+            bin/ocdev project validate "$project" --json
+          done

.gitignore

@@ -6,7 +6,17 @@ cmd/ocdev/nim.cfg
 nimcache/
 
 # Build outputs
-bin/ocdev
+bin/
+__pycache__/
+
+# Local agent/runtime state and private environment files
+/.pi/
+/.ocdev/
+.env
+.env.*
+*.env
+!.env.example
+!*.env.example
 
 # Editor
 *.swp

Makefile

@@ -1,6 +1,14 @@
-.PHONY: all clean test test-list-json test-create-config
+.PHONY: all clean test test-list-json test-create-config test-recipes test-live dev-setup FORCE
 
 NIMFLAGS = -d:release --opt:size
+NIM_TEST_FLAGS ?= --hints:off
+TEST_SRC = cmd/ocdev/tests
+RECIPE_TEST_BINS = bin/test_recipes bin/test_cli_recipes bin/test_recipe_engine \
+	bin/fake_incus_cli bin/fake_incus_engine bin/recipe_engine_harness
+TEST_BINS = $(RECIPE_TEST_BINS) bin/test_all bin/test_list_json bin/fake_incus_list \
+	bin/test_process_support bin/process_probe bin/test_create_config bin/fake_incus_create_config
+# The recipe CLI includes a maintained YAML parser; keep its release budget explicit.
+MAX_BINARY_BYTES ?= 2097152
 
 all: bin/ocdev
 
@@ -10,29 +18,65 @@ bin/ocdev: cmd/ocdev/src/ocdev.nim cmd/ocdev/src/*.nim
 bin/ocdev-debug: cmd/ocdev/src/ocdev.nim cmd/ocdev/src/*.nim
 	cd cmd/ocdev && nim c -o:../../bin/ocdev-debug src/ocdev.nim
 
-test: bin/ocdev
-	nim c -r cmd/ocdev/tests/test_all.nim
-	./cmd/ocdev/tests/integration.sh
+# Always invoke Nim for tests: its cache tracks transitive Atlas dependencies
+# and compiler configuration that Make's source timestamps alone cannot cover.
+FORCE:
+
+# Controllers and fake executables inherit the project-local Atlas nim.cfg.
+bin/test_%: $(TEST_SRC)/test_%.nim FORCE
+	mkdir -p bin
+	nim c $(NIM_TEST_FLAGS) --out:$@ $<
+
+bin/fake_incus_%: $(TEST_SRC)/fixtures/fake_incus_%.nim FORCE
+	mkdir -p bin
+	nim c $(NIM_TEST_FLAGS) --out:$@ $<
+
+bin/recipe_engine_harness: $(TEST_SRC)/fixtures/recipe_engine_harness.nim FORCE
+	mkdir -p bin
+	nim c $(NIM_TEST_FLAGS) --path:cmd/ocdev/src --out:$@ $<
+
+bin/process_probe: $(TEST_SRC)/fixtures/process_probe.nim FORCE
+	mkdir -p bin
+	nim c $(NIM_TEST_FLAGS) --out:$@ $<
+
+# Default tests use Nim only and never contact a live Incus daemon.
+test: bin/ocdev $(TEST_BINS)
+	bin/test_all
+	bin/test_process_support "$(abspath bin/process_probe)"
+	bin/test_recipes
+	bin/test_list_json "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_list)"
+	bin/test_create_config "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_create_config)"
+	bin/test_cli_recipes "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_cli)"
+	bin/test_recipe_engine "$(abspath bin/recipe_engine_harness)" "$(abspath bin/fake_incus_engine)"
+
+test-recipes: bin/ocdev $(RECIPE_TEST_BINS)
+	bin/test_recipes
+	bin/test_cli_recipes "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_cli)"
+	bin/test_recipe_engine "$(abspath bin/recipe_engine_harness)" "$(abspath bin/fake_incus_engine)"
+
+test-live: bin/ocdev
+	@test "$$OCDEV_LIVE_TESTS" = 1 || (echo 'Set OCDEV_LIVE_TESTS=1 to authorize live container operations'; exit 1)
+	bash cmd/ocdev/tests/recipe_smoke.sh
 
 # Read-only CLI coverage using fake Incus (no daemon required).
-test-list-json: bin/ocdev
-	python3 cmd/ocdev/tests/test_list_json.py ./bin/ocdev
+test-list-json: bin/ocdev bin/test_list_json bin/fake_incus_list
+	bin/test_list_json "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_list)"
 
-test-create-config: bin/ocdev
-	python3 cmd/ocdev/tests/test_create_config.py ./bin/ocdev
+test-create-config: bin/ocdev bin/test_create_config bin/fake_incus_create_config
+	bin/test_create_config "$(abspath bin/ocdev)" "$(abspath bin/fake_incus_create_config)"
 
 clean:
-	rm -f bin/ocdev bin/ocdev-debug
+	rm -f bin/ocdev bin/ocdev-debug bin/test-all bin/test-recipes $(TEST_BINS)
 	rm -rf cmd/ocdev/deps/pkgs
 	find cmd/ocdev -name "*.o" -delete
 
 # Development helpers
 dev-setup:
-	cd cmd/ocdev && atlas init && atlas use cligen
+	cd cmd/ocdev && atlas init && atlas install
 
 size-check: bin/ocdev
 	@echo "Binary size: $$(du -h bin/ocdev | cut -f1)"
 	@size=$$(stat -c%s bin/ocdev 2>/dev/null || stat -f%z bin/ocdev); \
-	if [ $$size -gt 512000 ]; then \
-		echo "WARNING: Binary exceeds 500KB target"; \
+	if [ $$size -gt $(MAX_BINARY_BYTES) ]; then \
+		echo "WARNING: Binary exceeds $(MAX_BINARY_BYTES)-byte target"; \
 	fi

README.md

@@ -14,6 +14,9 @@ This is especially useful when you need to run **complex projects requiring mult
 - **Docker-in-Docker** - Full Docker support via Incus nesting
 - **Low overhead** - ~100-200MB RAM per container vs 512MB+ for VMs
 - **Custom setup scripts** - Run post-create scripts to install additional tools
+- **Snapshot recipes and projects** - Reusable setup tasks, hooks, private file references, and pinned configuration
+- **Task and service operations** - Execution history, bounded logs, and optional process-compose control
+- **Automation** - `--json` on noninteractive management commands; existing `list --json` stays compatible
 
 ## Prerequisites
 
@@ -37,13 +40,34 @@ This is especially useful when you need to run **complex projects requiring mult
 
 ## Installation
 
+Build from source with Nim and Atlas installed (release binaries need neither).
+Atlas installs the dependencies declared in `cmd/ocdev/ocdev.nimble` into the
+project-local `cmd/ocdev/deps/` directory:
+
+```bash
+make dev-setup
+make
+```
+
 From the repository root:
 
 ```bash
 mkdir -p ~/.local/bin
 ln -sf "$(pwd)/bin/ocdev" ~/.local/bin/ocdev
 ```
 
+## Testing
+
+After `make dev-setup`, run `make test` on Linux. The unit and integration tests
+are written in Nim and use compiled fake Incus executables; they require neither
+Python nor a live Incus daemon. Test binaries are built into `bin/` using the
+same project-local Atlas dependencies as ocdev.
+
+Use `make test-list-json` for listing contracts, `make test-create-config` for
+create defaults and `--fresh`, or `make test-recipes` for recipe
+coverage. Live container tests remain a separate, explicitly authorized workflow;
+see [verification](docs/recipes.md#verification).
+
 ## Usage
 
 ```bash
@@ -87,6 +111,22 @@ ocdev delete myproject
 ocdev ports
 ```
 
+## Recipes and projects
+
+Recipes clone an **existing** `container/snapshot`, then deliver project files and run declared tasks/hooks. There is no new image builder, snapshot manager, or server.
+
+```bash
+ocdev recipe validate ./recipe.yaml --json
+ocdev create demo --project ./project.yaml --dry-run --json
+ocdev create demo --project ./project.yaml --json
+ocdev task run demo test --json
+ocdev services list demo --json
+ocdev runs list demo --json
+ocdev delete demo --dry-run --json
+```
+
+See **[Recipes and projects](docs/recipes.md)** for the schema, complete CLI, generic examples, JSON contracts, failure recovery, and security boundaries. Recipes are trusted executable configuration; a snapshot may inherit host mounts or contain credentials. No application-specific, cloud, or agent integration is required.
+
 ## Commands
 
 | Command | Description |
@@ -300,7 +340,7 @@ source ~/.nvm/nvm.sh
 nvm install 20
 ```
 
-If the post-create script fails, the container is kept so you can debug:
+If the post-create script fails, creation now exits nonzero and the container and its port allocation are kept so you can debug:
 
 ```bash
 ocdev shell myproject  # Debug what went wrong

cmd/ocdev/ocdev.nimble

@@ -6,3 +6,5 @@ srcDir        = "src"
 bin           = @["ocdev"]
 requires "nim >= 2.0.0"
 requires "cligen >= 1.7.0"
+requires "yaml >= 2.2.1"
+requires "checksums >= 0.2.2"

cmd/ocdev/src/automation.nim

@@ -0,0 +1,140 @@
+## Public, secret-minimizing CLI projections and legacy output isolation.
+import std/[os, osproc, json, strutils, posix]
+import config, ports, container
+
+proc captureLegacy*(body: proc(): int {.closure.}): int =
+  ## Reuse lifecycle procedures directly. Child output must never corrupt JSON.
+  flushFile(stdout)
+  flushFile(stderr)
+  let outFd = dup(STDOUT_FILENO)
+  let errFd = dup(STDERR_FILENO)
+  let sink = posix.open("/dev/null", O_WRONLY)
+  if outFd < 0 or errFd < 0 or sink < 0:
+    if outFd >= 0: discard posix.close(outFd)
+    if errFd >= 0: discard posix.close(errFd)
+    if sink >= 0: discard posix.close(sink)
+    raise newException(IOError, "output.capture_failed")
+  try:
+    if dup2(sink, STDOUT_FILENO) < 0 or dup2(sink, STDERR_FILENO) < 0:
+      raise newException(IOError, "output.capture_failed")
+    result = body()
+  finally:
+    flushFile(stdout)
+    flushFile(stderr)
+    discard dup2(outFd, STDOUT_FILENO)
+    discard dup2(errFd, STDERR_FILENO)
+    discard posix.close(outFd)
+    discard posix.close(errFd)
+    discard posix.close(sink)
+
+proc requireName*(name: string) =
+  if not validateName(name).valid:
+    raise newException(ValueError, "name.invalid: use a letter followed by letters, numbers or hyphens (max 50)")
+
+proc queryInstances*(): JsonNode =
+  let (raw, code) = execCmdEx("incus list --format=json 2>/dev/null")
+  if code != 0: raise newException(IOError, "incus.query_failed")
+  try:
+    result = parseJson(raw)
+    if result.kind != JArray: raise newException(ValueError, "invalid")
+    for item in result:
+      if item.kind != JObject or not item.hasKey("name") or item["name"].kind != JString or
+          not item.hasKey("status") or item["status"].kind != JString:
+        raise newException(ValueError, "invalid")
+  except CatchableError:
+    raise newException(IOError, "incus.invalid_metadata")
+
+proc publicInstance*(item: JsonNode): JsonNode =
+  let instance = item["name"].getStr()
+  if not instance.startsWith(ContainerPrefix):
+    raise newException(ValueError, "instance.unmanaged")
+  let name = instance[ContainerPrefix.len .. ^1]
+  requireName(name)
+  var uuid = newJNull()
+  if item.hasKey("config") and item["config"].kind == JObject:
+    let value = item["config"].getOrDefault("volatile.uuid")
+    if not value.isNil and value.kind == JString and value.getStr().len > 0: uuid = value
+  let port = getPort(name)
+  result = %*{"name": name, "instance": instance, "status": item["status"],
+    "uuid": uuid, "ssh_port": (if port > 0: %port else: newJNull())}
+
+proc plainInspect*(name: string): JsonNode =
+  requireName(name)
+  for item in queryInstances():
+    if item["name"].getStr() == ContainerPrefix & name:
+      result = publicInstance(item)
+      result["recipe"] = newJNull()
+      return
+  raise newException(ValueError, "environment.not_found")
+
+proc portRows*(): JsonNode =
+  result = newJArray()
+  let instances = queryInstances()
+  if not fileExists(PortsFile): return
+  for line in readFile(PortsFile).splitLines():
+    if line.len == 0: continue
+    let parts = line.split(':')
+    if parts.len != 2: raise newException(ValueError, "ports.invalid_state")
+    var port: int
+    try: port = parseInt(parts[1])
+    except ValueError: raise newException(ValueError, "ports.invalid_state")
+    requireName(parts[0])
+    if port < 1 or getServicePortBase(port) + ServicePortsCount - 1 > 65535:
+      raise newException(ValueError, "ports.invalid_state")
+    var status = "Missing"
+    for item in instances:
+      if item["name"].getStr() == ContainerPrefix & parts[0]: status = item["status"].getStr()
+    result.add(%*{"name": parts[0], "ssh_port": port,
+      "service_start": getServicePortBase(port),
+      "service_end": getServicePortBase(port) + ServicePortsCount - 1, "status": status})
+
+proc bindingRows*(name = ""): JsonNode =
+  if name.len > 0: requireName(name)
+  result = newJArray()
+  var found = name.len == 0
+  for item in queryInstances():
+    let fullName = item["name"].getStr()
+    if not fullName.startsWith(ContainerPrefix): continue
+    let shortName = fullName[ContainerPrefix.len .. ^1]
+    if name.len > 0 and name != shortName: continue
+    found = true
+    let devices = if item.hasKey("expanded_devices"): item["expanded_devices"]
+                  elif item.hasKey("devices"): item["devices"] else: newJObject()
+    if devices.kind != JObject: raise newException(ValueError, "incus.invalid_devices")
+    for key, device in devices:
+      if not key.startsWith("dyn-"): continue
+      try:
+        if device.kind != JObject or device.getOrDefault("type").getStr() != "proxy":
+          raise newException(ValueError, "invalid")
+        let host = parseInt(key[4 .. ^1])
+        let target = device.getOrDefault("connect").getStr()
+        if not target.startsWith("tcp:127.0.0.1:"):
+          raise newException(ValueError, "invalid")
+        let guest = parseInt(target.split(':')[^1])
+        if host < 1 or host > 65535 or guest < 1 or guest > 65535:
+          raise newException(ValueError, "invalid")
+        result.add(%*{"name": shortName, "host_port": host, "container_port": guest,
+          "status": item["status"]})
+      except CatchableError:
+        raise newException(ValueError, "ports.invalid_binding")
+  if not found: raise newException(ValueError, "environment.not_found")
+
+proc doctorResult*(): JsonNode =
+  var checks = newJArray()
+  let installed = findExe("incus").len > 0
+  checks.add(%*{"id": "incus.executable", "ok": installed})
+  var healthy = installed
+  if installed:
+    try:
+      let instances = queryInstances()
+      checks.add(%*{"id": "incus.connection", "ok": true, "instance_count": instances.len})
+      let allocations = portRows()
+      var stale = newJArray()
+      for item in allocations:
+        if item["status"].getStr() == "Missing": stale.add(item["name"])
+      checks.add(%*{"id": "ports.allocations", "ok": stale.len == 0, "missing_instances": stale})
+      if stale.len > 0: healthy = false
+    except CatchableError:
+      checks.add(%*{"id": "incus.metadata_and_ports", "ok": false})
+      healthy = false
+  %*{"ok": healthy, "checks": checks}