From d26d8cb6d71d0f4ee025b9567c20b58efcd7bc5d Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 18:27:48 +0100 Subject: [PATCH 01/21] Fix pool issues --- cli/cmd/coflux/pools.go | 206 +++--- cli/internal/api/client.go | 5 +- docs/docs/cli_reference.md | 6 +- docs/docs/pools.md | 19 +- server/lib/coflux/application.ex | 3 + server/lib/coflux/handlers/api.ex | 78 ++- server/lib/coflux/launchers/docker.ex | 23 +- server/lib/coflux/launchers/process.ex | 200 +++++- server/lib/coflux/orchestration/epoch.ex | 44 +- server/lib/coflux/orchestration/server.ex | 73 +- .../lib/coflux/orchestration/server/fleet.ex | 41 +- .../coflux/orchestration/server/rotation.ex | 8 + .../coflux/orchestration/server/routing.ex | 23 +- .../coflux/orchestration/server/scheduler.ex | 643 ++++++++++++------ .../coflux/orchestration/server/snapshots.ex | 23 +- .../lib/coflux/orchestration/server/state.ex | 28 +- server/lib/coflux/orchestration/workers.ex | 76 ++- server/lib/coflux/orchestration/workspaces.ex | 25 +- server/lib/coflux/topics/pool.ex | 32 + tests/support/adapter.py | 8 + tests/support/cli.py | 28 + tests/support/server.py | 4 +- tests/test_pools.py | 189 +++++ 23 files changed, 1348 insertions(+), 437 deletions(-) diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 7c3db246..7d6a28d0 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -551,14 +551,14 @@ func init() { poolsCreateCmd.Flags().String("type", "", "Launcher type (kubernetes, docker, process)") _ = poolsCreateCmd.MarkFlagRequired("type") poolsCreateCmd.Flags().StringArray("set", nil, "Set a field value (key=value)") - poolsCreateCmd.Flags().StringSlice("modules", nil, "Modules to be hosted") + poolsCreateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted") poolsCreateCmd.Flags().StringSlice("provides", nil, "Features that workers provide") poolsCreateCmd.Flags().StringSlice("accepts", nil, "Tags that executions must have") // pools update flags poolsUpdateCmd.Flags().StringArray("set", nil, "Set a field value (key=value)") poolsUpdateCmd.Flags().StringArray("unset", nil, "Unset a field") - poolsUpdateCmd.Flags().StringSlice("modules", nil, "Modules to be hosted") + poolsUpdateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted") poolsUpdateCmd.Flags().StringSlice("provides", nil, "Features that workers provide") poolsUpdateCmd.Flags().StringSlice("accepts", nil, "Tags that executions must have") poolsUpdateCmd.Flags().Bool("no-provides", false, "Clear provides") @@ -623,107 +623,70 @@ type poolFieldOp struct { value any // only for "set" } -// collectFieldOps builds an ordered list of field operations from os.Args, -// processing convenience flags (--modules, --provides, --accepts, --no-provides, -// --no-accepts) alongside --set/--unset in the order they appear on the command line. +// collectFieldOps builds the list of field operations from the parsed +// flags: the convenience flags (--modules, --provides, --accepts) and +// --set first, then --unset and the --no-* flags, so a field named by +// both ends up unset however it was ordered on the command line. +// +// Taking the order from os.Args instead would look more faithful, but it +// only sees the flag spellings the scan anticipates - a shorthand, or a +// form it doesn't recognise, is silently dropped - and order only decides +// anything for contradictory operations on one field. func collectFieldOps(cmd *cobra.Command) ([]poolFieldOp, error) { var ops []poolFieldOp - // Build a map from flag to its parsed values for quick lookup + if cmd.Flags().Changed("modules") { + values, _ := cmd.Flags().GetStringSlice("modules") + ops = append(ops, poolFieldOp{action: "set", key: "modules", value: toAnySlice(values)}) + } + + if cmd.Flags().Changed("provides") { + values, _ := cmd.Flags().GetStringSlice("provides") + ops = append(ops, poolFieldOp{action: "set", key: "provides", value: parseProvides(values)}) + } + + if cmd.Flags().Changed("accepts") { + values, _ := cmd.Flags().GetStringSlice("accepts") + ops = append(ops, poolFieldOp{action: "set", key: "accepts", value: parseProvides(values)}) + } + setValues, _ := cmd.Flags().GetStringArray("set") - unsetValues, _ := cmd.Flags().GetStringArray("unset") - setIdx := 0 - unsetIdx := 0 - - modulesValues, _ := cmd.Flags().GetStringSlice("modules") - providesValues, _ := cmd.Flags().GetStringSlice("provides") - acceptsValues, _ := cmd.Flags().GetStringSlice("accepts") - - // Walk os.Args to determine command-line order of flags - args := os.Args - for i := 0; i < len(args); i++ { - arg := args[i] - - // Normalize: handle --flag=value and --flag value forms - flagName := "" - switch { - case strings.HasPrefix(arg, "--set="): - flagName = "set" - case arg == "--set" && i+1 < len(args): - flagName = "set" - case strings.HasPrefix(arg, "--unset="): - flagName = "unset" - case arg == "--unset" && i+1 < len(args): - flagName = "unset" - case strings.HasPrefix(arg, "--modules=") || arg == "--modules": - flagName = "modules" - case strings.HasPrefix(arg, "--provides=") || arg == "--provides": - flagName = "provides" - case strings.HasPrefix(arg, "--accepts=") || arg == "--accepts": - flagName = "accepts" - case arg == "--no-provides": - flagName = "no-provides" - case arg == "--no-accepts": - flagName = "no-accepts" - default: - continue + for _, kv := range setValues { + key, val, hasVal := strings.Cut(kv, "=") + if !hasVal { + return nil, fmt.Errorf("invalid --set value %q: must be key=value", kv) + } + if !isValidFieldName(key) { + return nil, fmt.Errorf("unknown field %q", key) } + ops = append(ops, poolFieldOp{action: "set", key: key, value: parseSetValue(val)}) + } - switch flagName { - case "set": - if setIdx < len(setValues) { - kv := setValues[setIdx] - setIdx++ - key, val, hasVal := strings.Cut(kv, "=") - if !hasVal { - return nil, fmt.Errorf("invalid --set value %q: must be key=value", kv) - } - if !isValidFieldName(key) { - return nil, fmt.Errorf("unknown field %q", key) - } - ops = append(ops, poolFieldOp{action: "set", key: key, value: parseSetValue(val)}) - } - // Skip next arg if it was --set value (not --set=value) - if !strings.Contains(arg, "=") { - i++ - } - case "unset": - if unsetIdx < len(unsetValues) { - key := unsetValues[unsetIdx] - unsetIdx++ - if !isValidFieldName(key) { - return nil, fmt.Errorf("unknown field %q", key) - } - ops = append(ops, poolFieldOp{action: "unset", key: key}) - } - if !strings.Contains(arg, "=") { - i++ - } - case "modules": - ops = append(ops, poolFieldOp{action: "set", key: "modules", value: toAnySlice(modulesValues)}) - if !strings.Contains(arg, "=") { - i++ - } - case "provides": - ops = append(ops, poolFieldOp{action: "set", key: "provides", value: parseProvides(providesValues)}) - if !strings.Contains(arg, "=") { - i++ - } - case "accepts": - ops = append(ops, poolFieldOp{action: "set", key: "accepts", value: parseProvides(acceptsValues)}) - if !strings.Contains(arg, "=") { - i++ - } - case "no-provides": - ops = append(ops, poolFieldOp{action: "unset", key: "provides"}) - case "no-accepts": - ops = append(ops, poolFieldOp{action: "unset", key: "accepts"}) + unsetValues, _ := cmd.Flags().GetStringArray("unset") + for _, key := range unsetValues { + if !isValidFieldName(key) { + return nil, fmt.Errorf("unknown field %q", key) } + ops = append(ops, poolFieldOp{action: "unset", key: key}) + } + + if boolFlag(cmd, "no-provides") { + ops = append(ops, poolFieldOp{action: "unset", key: "provides"}) + } + + if boolFlag(cmd, "no-accepts") { + ops = append(ops, poolFieldOp{action: "unset", key: "accepts"}) } return ops, nil } +// boolFlag reads a bool flag that a given command may not define. +func boolFlag(cmd *cobra.Command, name string) bool { + value, err := cmd.Flags().GetBool(name) + return err == nil && value +} + // toAnySlice converts []string to []any for JSON serialization. func toAnySlice(ss []string) []any { result := make([]any, len(ss)) @@ -823,6 +786,13 @@ func runPoolsCreate(cmd *cobra.Command, args []string) error { } fmt.Printf("Created pool '%s'.\n", name) + + // A pool is matched to executions by module, so one with no modules + // can never launch anything. + if modules, ok := pool["modules"].([]any); !ok || len(modules) == 0 { + fmt.Fprintf(os.Stderr, "Warning: pool '%s' has no modules, so it will not be used. Set some with: coflux pools update %s --modules ...\n", name, name) + } + return nil } @@ -972,21 +942,52 @@ func runPoolsEnable(cmd *cobra.Command, args []string) error { return nil } +// redactedSecret is what the server substitutes for a secret the caller +// didn't ask for. It is deliberately not a usable value, so a config +// exported without secrets can't be imported back over the real ones. +const redactedSecret = "" + +// redactedPools names the pools whose config still carries a redacted +// secret, so both export and import can say which ones and why. +func redactedPools(pools map[string]map[string]any) []string { + var names []string + for name, pool := range pools { + launcher, ok := pool["launcher"].(map[string]any) + if !ok { + continue + } + for _, value := range launcher { + if s, ok := value.(string); ok && s == redactedSecret { + names = append(names, name) + break + } + } + } + sort.Strings(names) + return names +} + // pools export var poolsExportOutput string var poolsExportOnly []string +var poolsExportIncludeSecrets bool var poolsExportCmd = &cobra.Command{ Use: "export", Short: "Export pool configuration", - Long: `Export all pool configurations for the workspace as TOML. Writes to stdout by default.`, - RunE: runPoolsExport, + Long: `Export all pool configurations for the workspace as TOML. Writes to stdout by default. + +Launcher secrets, such as a Kubernetes token, are redacted unless --include-secrets +is given. A redacted file is refused by 'pools import', so it cannot silently clear +the secrets it omits.`, + RunE: runPoolsExport, } func init() { poolsExportCmd.Flags().StringVarP(&poolsExportOutput, "output", "o", "", "Output file (default: stdout)") poolsExportCmd.Flags().StringSliceVar(&poolsExportOnly, "only", nil, "Export only named pools") + poolsExportCmd.Flags().BoolVar(&poolsExportIncludeSecrets, "include-secrets", false, "Include launcher secrets (such as Kubernetes tokens) in the output") } func runPoolsExport(cmd *cobra.Command, args []string) error { @@ -1005,7 +1006,7 @@ func runPoolsExport(cmd *cobra.Command, args []string) error { return err } - result, err := client.GetPoolConfigs(cmd.Context(), workspaceID) + result, err := client.GetPoolConfigs(cmd.Context(), workspaceID, poolsExportIncludeSecrets) if err != nil { return err } @@ -1046,6 +1047,13 @@ func runPoolsExport(cmd *cobra.Command, args []string) error { fmt.Print(buf.String()) } + // Better to hear this now than when an import is refused later. + if names := redactedPools(pools); len(names) > 0 { + fmt.Fprintf(os.Stderr, + "Warning: secrets were redacted for %s, so this is not a complete configuration and cannot be imported as-is.\nRe-run with --include-secrets to export it in full.\n", + strings.Join(names, ", ")) + } + return nil } @@ -1122,6 +1130,12 @@ func runPoolsImport(cmd *cobra.Command, args []string) error { desiredPools[name] = tomlPoolToAPI(pool) } + if names := redactedPools(desiredPools); len(names) > 0 { + return fmt.Errorf( + "%s: secrets were redacted when this was exported, so importing it would clear them.\nRe-export with --include-secrets, or set the secret explicitly with: coflux pools update --set =", + strings.Join(names, ", ")) + } + // Connect and get current state workspace, err := requireWorkspace() if err != nil { @@ -1138,7 +1152,9 @@ func runPoolsImport(cmd *cobra.Command, args []string) error { return err } - result, err := client.GetPoolConfigs(cmd.Context(), workspaceID) + // With secrets: out-of-scope pools are merged into the update below, so + // redacted values here would overwrite the real ones on the server. + result, err := client.GetPoolConfigs(cmd.Context(), workspaceID, true) if err != nil { return err } diff --git a/cli/internal/api/client.go b/cli/internal/api/client.go index 544de8db..2a69427b 100644 --- a/cli/internal/api/client.go +++ b/cli/internal/api/client.go @@ -260,8 +260,9 @@ type GetPoolConfigsResult struct { } // GetPoolConfigs retrieves all pool configs for a workspace along with an ETag. -func (c *Client) GetPoolConfigs(ctx context.Context, workspaceID string) (*GetPoolConfigsResult, error) { - body := map[string]any{"workspaceId": workspaceID} +// Secrets (such as a Kubernetes token) are redacted unless includeSecrets is set. +func (c *Client) GetPoolConfigs(ctx context.Context, workspaceID string, includeSecrets bool) (*GetPoolConfigsResult, error) { + body := map[string]any{"workspaceId": workspaceID, "includeSecrets": includeSecrets} var pools map[string]map[string]any headers, err := c.post(ctx, "/api/get_pools", body, &pools) if err != nil { diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index 81fdf910..bf3cbf2f 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -181,7 +181,7 @@ All manifest commands accept `--adapter` to specify the adapter command. | `pools disable ` | Disable a pool (drain workers) | | `pools enable ` | Re-enable a pool | | `pools launches [worker-id]` | View launched workers (`--watch`) | -| `pools export` | Export pool configs as TOML (`-o`, `--only`) | +| `pools export` | Export pool configs as TOML (`-o`, `--only`, `--include-secrets`) | | `pools import [file]` | Import pool configs from TOML or stdin. Declarative: pools absent from the file are deleted. Flags: `--dry-run`, `--yes`, `--only` | ### Pool creation flags @@ -190,7 +190,7 @@ All manifest commands accept `--adapter` to specify the adapter command. |------|-------------| | `--type` | Launcher type: `kubernetes`, `docker`, `process` (required) | | `--set` | Set a field (e.g., `--set image=myapp:latest`, `--set env.KEY=VALUE`) | -| `--modules` | Modules to host | +| `--modules`, `-m` | Modules to host | | `--provides` | Features workers provide | | `--accepts` | Tags executions must have | @@ -200,7 +200,7 @@ All manifest commands accept `--adapter` to specify the adapter command. |------|-------------| | `--set` | Set a field | | `--unset` | Unset a field | -| `--modules` | Modules to host | +| `--modules`, `-m` | Modules to host | | `--provides` / `--no-provides` | Set or clear provides | | `--accepts` / `--no-accepts` | Set or clear accepts | diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 5c03b085..984f0958 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -31,6 +31,12 @@ coflux pools create mypool --type docker \ |-------|-------------| | `image` | Docker image to run | | `dockerHost` | Docker host (default: local socket) | +| `networkMode` | Container network mode (default: `host`) | + +The default `host` network mode lets workers reach a server running on the same +machine without further configuration. It isn't available on Docker Desktop, +where containers run inside a VM — there, set `networkMode` to `bridge` and +point `serverHost` at `host.docker.internal`. #### Process launcher @@ -81,7 +87,7 @@ Note that the token is stored in the orchestration database. | `namespace` | Kubernetes namespace (default: `default`) | | `apiServer` | Kubernetes API server URL (default: in-cluster) | | `token` | Bearer token for API authentication | -| `caCert` | CA certificate for TLS verification | +| `caCert` | Path, on the server's host, to a CA certificate file for TLS verification | | `insecure` | Skip TLS verification | | `serviceAccount` | Service account for launched pods | | `imagePullPolicy` | Image pull policy (`Always`, `IfNotPresent`, `Never`) | @@ -102,7 +108,7 @@ These fields apply to all launcher types: | Field / Flag | Description | |--------------|-------------| -| `--modules`, `-m` | Modules to host (can be specified multiple times) | +| `--modules`, `-m` | Modules to host (can be specified multiple times). Module names only — wildcards aren't supported, since this is also what launched workers are told to import | | `--provides` | Features that workers provide (e.g., `gpu:A100`) | | `--accepts` | Tags that executions must have to be assigned to this pool | | `serverHost` | Server host override for launched workers | @@ -154,6 +160,15 @@ coflux pools export --only mypool --only gpu-pool -o pools.toml coflux pools import pools.toml ``` +Launcher secrets — currently the Kubernetes `token` — are redacted on export +unless `--include-secrets` is given. Importing a redacted file is refused rather +than silently clearing the secrets it omits, so use `--include-secrets` when the +exported file is meant to be imported again: + +```bash +coflux pools export --include-secrets -o pools.toml +``` + ## Provides, accepts, and requires Workers can declare features they _provide_, and targets can _require_ specific features. This allows routing executions to appropriate workers — for example, GPU-intensive tasks to GPU-equipped workers. diff --git a/server/lib/coflux/application.ex b/server/lib/coflux/application.ex index bf2cd0f6..f42e19c6 100644 --- a/server/lib/coflux/application.ex +++ b/server/lib/coflux/application.ex @@ -13,6 +13,9 @@ defmodule Coflux.Application do # TODO: separate launch supervisor per project? (and specify max_children?) {Task.Supervisor, name: Coflux.LauncherSupervisor}, {DynamicSupervisor, name: Coflux.ProcessLauncher.Supervisor, strategy: :one_for_one}, + # Keyed by OS pid, so a launched worker is found by something that + # means the same thing after a restart as before one. + {Registry, keys: :unique, name: Coflux.ProcessLauncher.Registry}, Orchestration.Supervisor, {Registry, keys: :unique, name: Coflux.Logs.Registry}, Logs.Supervisor, diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index eb493709..dd7ed5fb 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -6,6 +6,11 @@ defmodule Coflux.Handlers.Api do @max_parameters 20 + # Stands in for a secret the caller didn't ask for. It is deliberately + # not a valid value: a config exported without secrets and imported + # again is refused rather than quietly clearing the real one. + @redacted_secret "" + # A directory upload arrives as one entry per file, so this bounds an # accidental drop of a very large tree. Unlike the sizes, which the # client asserts, the count is something the server can see for itself. @@ -305,7 +310,7 @@ defmodule Coflux.Handlers.Api do defp handle(req, "POST", ["disable_pool"], project_id, access) do case read_arguments(req, %{ workspace_id: "workspaceId", - pool_name: "poolName" + pool_name: {"poolName", &parse_pool_name/1} }) do {:ok, arguments, req} -> case Orchestration.disable_pool( @@ -315,6 +320,7 @@ defmodule Coflux.Handlers.Api do access ) do :ok -> :cowboy_req.reply(204, req) + {:error, :not_found} -> json_error_response(req, "not_found", status: 404) {:error, :workspace_invalid} -> json_error_response(req, "not_found", status: 404) {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) end @@ -327,7 +333,7 @@ defmodule Coflux.Handlers.Api do defp handle(req, "POST", ["enable_pool"], project_id, access) do case read_arguments(req, %{ workspace_id: "workspaceId", - pool_name: "poolName" + pool_name: {"poolName", &parse_pool_name/1} }) do {:ok, arguments, req} -> case Orchestration.enable_pool( @@ -337,6 +343,7 @@ defmodule Coflux.Handlers.Api do access ) do :ok -> :cowboy_req.reply(204, req) + {:error, :not_found} -> json_error_response(req, "not_found", status: 404) {:error, :workspace_invalid} -> json_error_response(req, "not_found", status: 404) {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) end @@ -347,13 +354,19 @@ defmodule Coflux.Handlers.Api do end defp handle(req, "POST", ["get_pools"], project_id, _access) do - case read_arguments(req, %{workspace_id: "workspaceId"}) do + case read_arguments( + req, + %{workspace_id: "workspaceId"}, + %{include_secrets: {"includeSecrets", &parse_boolean(&1, optional: true)}} + ) do {:ok, arguments, req} -> + include_secrets = Map.get(arguments, :include_secrets) == true + case Orchestration.get_pools(project_id, arguments.workspace_id) do {:ok, pools, hash} -> result = Map.new(pools, fn {name, pool} -> - {name, build_pool_config(pool)} + {name, build_pool_config(pool, include_secrets)} end) req = :cowboy_req.set_resp_header("etag", "\"#{hash}\"", req) @@ -1086,20 +1099,6 @@ defmodule Coflux.Handlers.Api do end end - defp is_valid_module_pattern?(pattern) do - cond do - not is_binary(pattern) -> - false - - String.length(pattern) > 100 -> - false - - true -> - parts = String.split(pattern, ".") - Enum.all?(parts, &(&1 == "*" || Regex.match?(~r/^[a-z_][a-z0-9_]*$/i, &1))) - end - end - defp is_valid_tag_key?(key) do is_valid_string?(key, regex: ~r/^[a-z0-9_-]{1,20}$/i) end @@ -1120,10 +1119,15 @@ defmodule Coflux.Handlers.Api do end end + # A pool's modules are module names, not patterns: the same list is + # handed to the launcher as the worker's arguments, so a wildcard would + # be passed to the worker to import - nothing expands it - as well as + # matching no execution. Accepting one would mean a pool that quietly + # never runs anything, so they are validated like any other module name. defp parse_modules(value) do value = List.wrap(value) - if Enum.all?(value, &is_valid_module_pattern?/1) do + if Enum.all?(value, &is_valid_module_name?/1) do {:ok, value} else {:error, :invalid} @@ -1173,6 +1177,7 @@ defmodule Coflux.Handlers.Api do defp parse_docker_launcher(value) do image = Map.get(value, "image") docker_host = Map.get(value, "dockerHost") + network_mode = Map.get(value, "networkMode") cond do not is_binary(image) or String.length(image) > 200 -> @@ -1181,12 +1186,19 @@ defmodule Coflux.Handlers.Api do not is_nil(docker_host) and (not is_binary(docker_host) or String.length(docker_host) > 200) -> {:error, :invalid} + not is_nil(network_mode) and + (not is_binary(network_mode) or String.length(network_mode) > 200) -> + {:error, :invalid} + true -> launcher = %{type: :docker, image: image} launcher = if docker_host, do: Map.put(launcher, :docker_host, docker_host), else: launcher + launcher = + if network_mode, do: Map.put(launcher, :network_mode, network_mode), else: launcher + {:ok, launcher} end end @@ -1242,6 +1254,9 @@ defmodule Coflux.Handlers.Api do not is_nil(token) and not is_binary(token) -> {:error, :invalid} + token == @redacted_secret -> + {:error, :redacted} + not is_nil(ca_cert) and not is_binary(ca_cert) -> {:error, :invalid} @@ -1450,12 +1465,18 @@ defmodule Coflux.Handlers.Api do {:ok, pool} when is_map(pool) -> {:cont, {:ok, Map.put(result, name, pool)}} + # Keep why, against the pool it came from: "invalid" alone + # leaves the caller no idea which pool, or what to do about + # it - and `redacted` in particular has a specific remedy. + {:error, error} -> + {:halt, {:error, %{name => error}}} + _ -> - {:halt, {:error, :invalid}} + {:halt, {:error, %{name => :invalid}}} end {:error, _} -> - {:halt, {:error, :invalid}} + {:halt, {:error, %{name => :invalid_name}}} end end) @@ -1464,7 +1485,11 @@ defmodule Coflux.Handlers.Api do end end - defp build_pool_config(pool) do + defp secret_value(nil, _include_secrets), do: nil + defp secret_value(value, true), do: value + defp secret_value(_value, false), do: @redacted_secret + + defp build_pool_config(pool, include_secrets) do provides = pool.provides accepts = Map.get(pool, :accepts, %{}) @@ -1474,18 +1499,19 @@ defmodule Coflux.Handlers.Api do config = if Enum.any?(accepts), do: Map.put(config, "accepts", accepts), else: config if pool.launcher do - Map.put(config, "launcher", build_launcher_config(pool.launcher)) + Map.put(config, "launcher", build_launcher_config(pool.launcher, include_secrets)) else config end end - defp build_launcher_config(launcher) do + defp build_launcher_config(launcher, include_secrets) do type_fields = case launcher.type do :docker -> %{"type" => "docker", "image" => launcher.image} |> maybe_put_value("dockerHost", Map.get(launcher, :docker_host)) + |> maybe_put_value("networkMode", Map.get(launcher, :network_mode)) :process -> %{"type" => "process", "directory" => launcher.directory} @@ -1495,7 +1521,7 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("namespace", Map.get(launcher, :namespace)) |> maybe_put_value("apiServer", Map.get(launcher, :api_server)) |> maybe_put_value("serviceAccount", Map.get(launcher, :service_account)) - |> maybe_put_value("token", Map.get(launcher, :token)) + |> maybe_put_value("token", secret_value(Map.get(launcher, :token), include_secrets)) |> maybe_put_value("caCert", Map.get(launcher, :ca_cert)) |> maybe_put_value("insecure", Map.get(launcher, :insecure)) |> maybe_put_value("imagePullPolicy", Map.get(launcher, :image_pull_policy)) @@ -1631,6 +1657,7 @@ defmodule Coflux.Handlers.Api do field_specs = [ {"image", &is_binary/1}, {"dockerHost", &is_binary/1}, + {"networkMode", &is_binary/1}, {"directory", &is_binary/1}, {"namespace", &is_binary/1}, {"serviceAccount", &is_binary/1}, @@ -1662,6 +1689,7 @@ defmodule Coflux.Handlers.Api do key_map = %{ "image" => :image, "dockerHost" => :docker_host, + "networkMode" => :network_mode, "directory" => :directory, "namespace" => :namespace, "serviceAccount" => :service_account, diff --git a/server/lib/coflux/launchers/docker.ex b/server/lib/coflux/launchers/docker.ex index 53b677ae..3d5076b9 100644 --- a/server/lib/coflux/launchers/docker.ex +++ b/server/lib/coflux/launchers/docker.ex @@ -3,17 +3,25 @@ defmodule Coflux.DockerLauncher do @log_tail_lines 20 @log_max_bytes 1024 + # Host networking is the default because it is the one mode in which a + # worker can reach a server running on the same machine without knowing + # anything about how to address it. It isn't available everywhere - + # Docker Desktop runs containers in a VM, where the host network is the + # VM's - so it is a default, not a fixture. + @default_network_mode "host" + def launch(env, modules, config, _opts \\ %{}) do docker_conn = parse_docker_host(config[:docker_host]) container_env = Enum.map(env, fn {k, v} -> "#{k}=#{v}" end) + network_mode = Map.get(config, :network_mode) || @default_network_mode with {:ok, %{"Id" => container_id}} <- create_container( docker_conn, %{ "Image" => Map.fetch!(config, :image), - "HostConfig" => %{"NetworkMode" => "host"}, + "HostConfig" => %{"NetworkMode" => network_mode}, "Cmd" => modules, "Env" => container_env } @@ -120,7 +128,7 @@ defmodule Coflux.DockerLauncher do 400 -> {:error, :bad_parameter} 404 -> {:error, :no_such_image} 409 -> {:error, :conflict} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> @@ -135,7 +143,7 @@ defmodule Coflux.DockerLauncher do 204 -> :ok 304 -> {:error, :container_already_started} 404 -> {:error, :no_such_container} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> @@ -149,7 +157,7 @@ defmodule Coflux.DockerLauncher do case response.status do 200 -> {:ok, response.body} 404 -> {:error, :no_such_container} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> @@ -172,7 +180,7 @@ defmodule Coflux.DockerLauncher do case response.status do 200 -> {:ok, demux_docker_logs(response.body)} 404 -> {:error, :no_such_container} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> @@ -189,6 +197,7 @@ defmodule Coflux.DockerLauncher do defp normalize_launch_error(:container_already_started), do: "launch_container_exists" defp normalize_launch_error(:no_such_container), do: "launch_container_not_found" defp normalize_launch_error(:request_failed), do: "launch_request_failed" + defp normalize_launch_error({:unexpected_status, status}), do: "launch_status:#{status}" defp normalize_launch_error(_), do: "launch_request_failed" defp demux_docker_logs(data) when is_binary(data) do @@ -213,7 +222,7 @@ defmodule Coflux.DockerLauncher do 204 -> :ok 304 -> :ok 404 -> {:error, :no_such_container} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> @@ -229,7 +238,7 @@ defmodule Coflux.DockerLauncher do 400 -> {:error, :bad_parameter} 404 -> {:error, :no_such_container} 409 -> {:error, :conflict} - 500 -> {:error, :server_error} + status -> {:error, {:unexpected_status, status}} end {:error, _} = error -> diff --git a/server/lib/coflux/launchers/process.ex b/server/lib/coflux/launchers/process.ex index 47cfea94..314d8003 100644 --- a/server/lib/coflux/launchers/process.ex +++ b/server/lib/coflux/launchers/process.ex @@ -1,7 +1,34 @@ defmodule Coflux.ProcessLauncher do + @moduledoc """ + Runs workers as processes on the server's own machine. + + What identifies a launched worker is its OS process, not the Erlang + process supervising the port. The server can be restarted while the + worker carries on running, and across that restart an Erlang pid means + nothing: it decodes into a pid of the *new* VM, which may well have been + reused by something unrelated. So the launch data is an OS pid and a + fingerprint of when that process started - the pair, because pid numbers + are reused too, and a recycled pid must not be mistaken for the worker + or signalled as if it were. + + The supervising process is found by looking the OS pid up in a registry + that only holds this VM's workers. When it is there it is preferred, + because it has the output to report; when it isn't - after a restart - + the OS pid alone still answers "is it alive?" and "stop". + """ + + @registry Coflux.ProcessLauncher.Registry + @log_tail_lines 20 @log_max_bytes 1024 + # How long a worker is given to exit after SIGTERM before SIGKILL. + @stop_grace_ms 10_000 + + # Exit codes that mean "it did what we asked": 128 + SIGTERM and + # 128 + SIGKILL. + @signalled_exit_codes [143, 137] + def launch(env, modules, config, _opts \\ %{}) do cli_path = Coflux.Config.cli_path() directory = Map.fetch!(config, :directory) @@ -29,38 +56,122 @@ defmodule Coflux.ProcessLauncher do {Coflux.ProcessLauncher.Worker, port_opts} ) do {:ok, pid} -> - {:ok, %{pid: pid}} + case GenServer.call(pid, :os_pid) do + {:ok, os_pid} -> + {:ok, %{os_pid: os_pid, started: process_started(os_pid)}} + + :error -> + {:error, "launch_process_failed"} + end {:error, _reason} -> {:error, "launch_process_failed"} end end - def stop(%{pid: pid}) do - if Process.alive?(pid) do - GenServer.call(pid, :stop) + def stop(data) do + case resolve(data) do + {:supervised, pid} -> + GenServer.call(pid, :stop) + :ok + + {:unsupervised, os_pid} -> + signal(os_pid, "TERM") + :ok + + # Already gone, which is what was being asked for. + :gone -> + :ok end + end - :ok + def poll(data) do + case resolve(data) do + {:supervised, pid} -> + case GenServer.call(pid, :status) do + :running -> + {:ok, true} + + {:exited, exit_status, stop_requested, output} -> + GenServer.stop(pid, :normal) + error = exit_error(exit_status, stop_requested) + logs = if error, do: format_logs(output) + {:ok, false, error, logs} + end + + {:unsupervised, _os_pid} -> + # Running, but launched before this server started, so there is no + # output to report if and when it exits. + {:ok, true} + + :gone -> + {:ok, false, "process_lost", nil} + end end - def poll(%{pid: pid}) do - if Process.alive?(pid) do - case GenServer.call(pid, :status) do - :running -> - {:ok, true} + # A stop asks for SIGTERM and escalates to SIGKILL, so exiting from + # either is the worker doing as it was told, not a failure. Any other + # non-zero exit during a drain is a real crash and is reported as one. + defp exit_error(0, _stop_requested), do: nil + defp exit_error(status, true) when status in @signalled_exit_codes, do: nil + defp exit_error(status, _stop_requested), do: "exit_code:#{status}" - {:exited, exit_status, output} -> - GenServer.stop(pid, :normal) - error = if exit_status != 0, do: "exit_code:#{exit_status}" - logs = if error, do: format_logs(output) - {:ok, false, error, logs} - end - else - {:ok, false, "process_lost", nil} + defp resolve(%{os_pid: os_pid} = data) do + cond do + # Only this VM's workers are in the registry, so finding one is + # proof that the process it names is the one that was launched. + pid = lookup(os_pid) -> + {:supervised, pid} + + process_alive?(os_pid) && process_started(os_pid) == Map.get(data, :started) -> + {:unsupervised, os_pid} + + true -> + :gone + end + end + + # Launch data from before the OS pid was recorded. Nothing that knows how + # to reach the process survives, so it is treated as gone rather than + # having an unrelated Erlang pid signalled on its behalf. + defp resolve(_data), do: :gone + + defp lookup(os_pid) do + case Registry.lookup(@registry, os_pid) do + [{pid, _}] -> pid + [] -> nil end end + defp process_alive?(os_pid) do + match?({_, 0}, System.cmd("kill", ["-0", Integer.to_string(os_pid)], stderr_to_stdout: true)) + end + + # When the process started, which is what distinguishes it from a later + # one that reused its pid. `ps` reports this on both Linux and macOS; + # nil where it can't be read, in which case the pid has to stand alone. + defp process_started(os_pid) do + case System.cmd("ps", ["-o", "lstart=", "-p", Integer.to_string(os_pid)], + stderr_to_stdout: true + ) do + {output, 0} -> + case String.trim(output) do + "" -> nil + value -> value + end + + {_output, _} -> + nil + end + catch + :error, _ -> nil + end + + defp signal(os_pid, name) do + System.cmd("kill", ["-#{name}", Integer.to_string(os_pid)], stderr_to_stdout: true) + :ok + end + defp format_logs(""), do: nil defp format_logs(content) do @@ -81,6 +192,9 @@ defmodule Coflux.ProcessLauncher do defp shell_escape(arg) do "'" <> String.replace(arg, "'", "'\\''") <> "'" end + + def registry, do: @registry + def stop_grace_ms, do: @stop_grace_ms end defmodule Coflux.ProcessLauncher.Worker do @@ -93,7 +207,25 @@ defmodule Coflux.ProcessLauncher.Worker do @impl true def init(port_opts) do port = Port.open({:spawn_executable, "/bin/sh"}, port_opts) - {:ok, %{port: port, output: [], exit_status: nil, stop_requested: false}} + + with {:os_pid, os_pid} <- Port.info(port, :os_pid), + # An exited worker's process lingers until a poll reports the + # exit, so the OS could in principle hand its pid to a new one + # first. Refusing to launch is the right answer: the pool retries. + {:ok, _} <- Registry.register(Coflux.ProcessLauncher.registry(), os_pid, nil) do + {:ok, + %{ + port: port, + os_pid: os_pid, + output: [], + exit_status: nil, + stop_requested: false, + kill_timer: nil + }} + else + nil -> {:stop, :no_os_pid} + {:error, {:already_registered, _}} -> {:stop, :os_pid_conflict} + end end @impl true @@ -102,26 +234,40 @@ defmodule Coflux.ProcessLauncher.Worker do end def handle_info({port, {:exit_status, status}}, %{port: port} = state) do - {:noreply, %{state | exit_status: status}} + if state.kill_timer, do: Process.cancel_timer(state.kill_timer) + {:noreply, %{state | exit_status: status, kill_timer: nil}} + end + + # A worker that ignored SIGTERM is killed outright rather than left + # running with nothing further going to happen to it. + def handle_info(:kill, %{exit_status: nil} = state) do + System.cmd("kill", ["-KILL", Integer.to_string(state.os_pid)], stderr_to_stdout: true) + {:noreply, %{state | kill_timer: nil}} end + def handle_info(:kill, state), do: {:noreply, %{state | kill_timer: nil}} + @impl true + def handle_call(:os_pid, _from, state) do + {:reply, {:ok, state.os_pid}, state} + end + def handle_call(:status, _from, state) do if is_nil(state.exit_status) do {:reply, :running, state} else logs = IO.iodata_to_binary(state.output) - # Treat as clean exit if stop was requested (even if exit code is non-zero - # due to SIGTERM producing exit code 143). - exit_status = if state.stop_requested, do: 0, else: state.exit_status - {:reply, {:exited, exit_status, logs}, state} + # The real exit code, with whether a stop was asked for alongside it + # rather than folded into it: a worker that crashed while draining + # should not look like one that shut down cleanly. + {:reply, {:exited, state.exit_status, state.stop_requested, logs}, state} end end def handle_call(:stop, _from, %{exit_status: nil} = state) do - {:os_pid, os_pid} = Port.info(state.port, :os_pid) - System.cmd("kill", [Integer.to_string(os_pid)], stderr_to_stdout: true) - {:reply, :ok, %{state | stop_requested: true}} + System.cmd("kill", ["-TERM", Integer.to_string(state.os_pid)], stderr_to_stdout: true) + timer = Process.send_after(self(), :kill, Coflux.ProcessLauncher.stop_grace_ms()) + {:reply, :ok, %{state | stop_requested: true, kill_timer: timer}} end def handle_call(:stop, _from, state) do diff --git a/server/lib/coflux/orchestration/epoch.ex b/server/lib/coflux/orchestration/epoch.ex index aa393ac2..19630577 100644 --- a/server/lib/coflux/orchestration/epoch.ex +++ b/server/lib/coflux/orchestration/epoch.ex @@ -965,13 +965,51 @@ defmodule Coflux.Orchestration.Epoch do if old_ws_ids == [] do %{} else - copy_pools_query(old_db, new_db, workspace_ids, old_ws_ids) + pool_ids = copy_pools_query(old_db, new_db, workspace_ids, old_ws_ids) + copy_pool_states(old_db, new_db, workspace_ids, old_ws_ids) + pool_ids end end + # Whether a pool is disabled lives apart from its definition, so without + # this every disabled pool comes back enabled in the new epoch. + defp copy_pool_states(old_db, new_db, workspace_ids, old_ws_ids) do + placeholders = Enum.map_join(1..length(old_ws_ids), ", ", &"?#{&1}") + + {:ok, rows} = + query( + old_db, + """ + SELECT workspace_id, pool_name, state, created_at, created_by + FROM pool_states + WHERE workspace_id IN (#{placeholders}) + ORDER BY rowid + """, + List.to_tuple(old_ws_ids) + ) + + Enum.each(rows, fn {old_ws_id, pool_name, state, created_at, created_by} -> + {:ok, _} = + insert_one(new_db, :pool_states, %{ + workspace_id: Map.fetch!(workspace_ids, old_ws_id), + pool_name: pool_name, + state: state, + created_at: created_at, + created_by: ensure_principal(old_db, new_db, created_by) + }) + end) + end + defp copy_pools_query(old_db, new_db, workspace_ids, old_ws_ids) do placeholders = Enum.map_join(1..length(old_ws_ids), ", ", &"?#{&1}") + # Every row, deletions included, in id order. A pool is a sequence of + # immutable rows and the current one is whichever has the highest id, + # so both parts matter: skipping the null-definition rows would + # resurrect deleted pools by promoting the definition they replaced, + # and copying out of order would change which row wins. Superseded + # rows are kept because workers reference the row they were launched + # from, not just the current one. {:ok, rows} = query( old_db, @@ -979,8 +1017,8 @@ defmodule Coflux.Orchestration.Epoch do SELECT id, external_id, workspace_id, name, pool_definition_id, created_at, created_by FROM pools - WHERE pool_definition_id IS NOT NULL - AND workspace_id IN (#{placeholders}) + WHERE workspace_id IN (#{placeholders}) + ORDER BY id """, List.to_tuple(old_ws_ids) ) diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index 23d267a0..46b01201 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -143,7 +143,11 @@ defmodule Coflux.Orchestration.Server do data: data, session_id: nil, stop_id: nil, - last_poll_at: nil + stop_retry_at: nil, + last_poll_at: nil, + polling: false, + poll_failures: 0, + first_poll_failure_at: nil }) end ) @@ -528,7 +532,10 @@ defmodule Coflux.Orchestration.Server do state = state.workers |> Enum.reduce(state, fn {worker_id, worker}, state -> - if worker.state == :active && + # Pool names are per workspace: a pool of the same name in + # another workspace is a different pool, and its workers are + # not ours to drain. + if worker.state == :active && worker.workspace_id == workspace_id && MapSet.member?(changed_pool_names, worker.pool_name) do Fleet.update_worker_state( state, @@ -619,11 +626,12 @@ defmodule Coflux.Orchestration.Server do pool_patch, access[:principal_id] ) do - {:ok, _pool_id} -> + {:ok, _pool_id, :updated} -> state = state.workers |> Enum.reduce(state, fn {worker_id, worker}, state -> - if worker.state == :active && worker.pool_name == pool_name do + if worker.state == :active && worker.workspace_id == workspace_id && + worker.pool_name == pool_name do Fleet.update_worker_state(state, worker_id, :draining, workspace_id, pool_name) else state @@ -644,6 +652,9 @@ defmodule Coflux.Orchestration.Server do {:reply, :ok, state} + {:ok, _pool_id, :unchanged} -> + {:reply, :ok, state} + {:error, :not_found} -> {:reply, {:error, :not_found}, state} @@ -658,13 +669,14 @@ defmodule Coflux.Orchestration.Server do defp dispatch_call({:disable_pool, workspace_external_id, pool_name, access}, state) do with {:ok, workspace_id, _} <- - Permissions.require_workspace(state, workspace_external_id, access) do + Permissions.require_workspace(state, workspace_external_id, access), + :ok <- require_pool(state, workspace_id, pool_name) do :ok = Workspaces.disable_pool(state.db, workspace_id, pool_name, access[:principal_id]) state = state |> put_in( - [Access.key(:pools), Access.key(workspace_id, %{}), Access.key(pool_name, %{}), :state], + [Access.key(:pools), Access.key!(workspace_id), Access.key!(pool_name), :state], :disabled ) |> Effects.emit(%PoolStateChanged{ @@ -684,13 +696,14 @@ defmodule Coflux.Orchestration.Server do defp dispatch_call({:enable_pool, workspace_external_id, pool_name, access}, state) do with {:ok, workspace_id, _} <- - Permissions.require_workspace(state, workspace_external_id, access) do + Permissions.require_workspace(state, workspace_external_id, access), + :ok <- require_pool(state, workspace_id, pool_name) do :ok = Workspaces.enable_pool(state.db, workspace_id, pool_name, access[:principal_id]) state = state |> put_in( - [Access.key(:pools), Access.key(workspace_id, %{}), Access.key(pool_name, %{}), :state], + [Access.key(:pools), Access.key!(workspace_id), Access.key!(pool_name), :state], :active ) |> Effects.emit(%PoolStateChanged{ @@ -874,6 +887,8 @@ defmodule Coflux.Orchestration.Server do worker_id: nil, last_idle_at: now, activated_at: nil, + declared_at: nil, + ready_deadline_at: nil, activation_timeout: activation_timeout, reconnection_timeout: reconnection_timeout, total_executions: 0 @@ -940,12 +955,26 @@ defmodule Coflux.Orchestration.Server do |> Enum.reverse() |> Enum.each(&send(pid, &1)) + # A worker that has connected but not yet said what it can run is + # given until this deadline to do so, after which it is treated as + # broken rather than idle (see `Scheduler`). Each connection gets a + # fresh one; a session that has already declared keeps none. + ready_deadline_at = + if is_nil(session.declared_at) do + System.os_time(:millisecond) + session.activation_timeout + end + state = state |> put_in([Access.key(:connections), ref], {pid, session_id}) |> update_in( [Access.key(:sessions), session_id], - &Map.merge(&1, %{connection: ref, queue: [], activated_at: activated_at}) + &Map.merge(&1, %{ + connection: ref, + queue: [], + activated_at: activated_at, + ready_deadline_at: ready_deadline_at + }) ) state = Effects.emit(state, Fleet.session_event(state, session)) @@ -992,11 +1021,26 @@ defmodule Coflux.Orchestration.Server do now = System.os_time(:millisecond) + previous = Map.fetch!(state.sessions, session_id) + state = state |> Fleet.assign_targets(targets, session_id) |> put_in([Access.key(:sessions), session_id, :concurrency], concurrency) |> put_in([Access.key(:sessions), session_id, :last_idle_at], now) + # The worker has answered, so it is ready and the deadline for + # answering no longer applies - even if it declared nothing, which + # is an empty manifest rather than a broken worker. + |> put_in([Access.key(:sessions), session_id, :declared_at], previous.declared_at || now) + |> put_in([Access.key(:sessions), session_id, :ready_deadline_at], nil) + + # A pool that produces a working worker has no failures to back off + # from, whatever its previous launches did. + state = + case previous.worker_id && Map.fetch(state.workers, previous.worker_id) do + {:ok, worker} -> Map.update!(state, :pool_failures, &Map.delete(&1, worker.pool_id)) + _ -> state + end session = Map.fetch!(state.sessions, session_id) @@ -2885,4 +2929,15 @@ defmodule Coflux.Orchestration.Server do defp ok_or({:ok, val}, _reason), do: {:ok, val} defp ok_or(:error, reason), do: {:error, reason} + + # Enabling or disabling a pool that doesn't exist must not conjure one: + # a state-only entry has no launcher and no modules, and every reader of + # `state.pools` assumes a pool has both. + defp require_pool(state, workspace_id, pool_name) do + if state.pools |> Map.get(workspace_id, %{}) |> Map.has_key?(pool_name) do + :ok + else + {:error, :not_found} + end + end end diff --git a/server/lib/coflux/orchestration/server/fleet.ex b/server/lib/coflux/orchestration/server/fleet.ex index 97d9a68f..a5a2327d 100644 --- a/server/lib/coflux/orchestration/server/fleet.ex +++ b/server/lib/coflux/orchestration/server/fleet.ex @@ -8,7 +8,9 @@ defmodule Coflux.Orchestration.Server.Fleet do and picks up where it left off - which is why a session expires on a timer rather than on disconnect. Two timers apply: a worker that never connects expires on its activation timeout, one that connected and went - away on its reconnection timeout. + away on its reconnection timeout. A third case is neither, and is + handled by `Scheduler`: a worker that connects but never declares any + targets is deactivated once its readiness deadline passes. A *pool* is a declaration that workers of some shape should exist, and the launcher is what makes them. Launching is asynchronous: the task is @@ -190,6 +192,21 @@ defmodule Coflux.Orchestration.Server.Fleet do end) end + @doc """ + Whether a session has ever been in a position to take work: it + connected, and it said what it can run. + + Until both have happened the session has never been able to accept an + execution, so the fact that it isn't running one says nothing about it + being surplus - which is why the idle timeout only applies from here. + Declaring an *empty* set of targets still counts: the worker answered, + it just has nothing to offer, and it should be allowed to drain like + any other rather than pinning its pool open forever. + """ + def session_ready?(session) do + !is_nil(session.activated_at) && !is_nil(session.declared_at) + end + def session_at_capacity?(session) do if session.concurrency != 0 do load = MapSet.size(session.starting) + MapSet.size(session.executing) @@ -405,7 +422,20 @@ defmodule Coflux.Orchestration.Server.Fleet do }) end - def deactivate_worker(state, worker_id, error, logs \\ nil) do + @doc """ + Retires a worker: no more work, no more polling, and its session gone. + + Deactivation can be reached twice for the same worker - two launcher + tasks landing on it, or a poll racing a stop - so a worker that has + already gone is not an error, just nothing left to do. + """ + def deactivate_worker(state, worker_id, error, logs \\ nil) + + def deactivate_worker(%{workers: workers} = state, worker_id, _error, _logs) + when not is_map_key(workers, worker_id), + do: state + + def deactivate_worker(state, worker_id, error, logs) do {:ok, deactivated_at} = Workers.create_worker_deactivation(state.db, worker_id, error, logs) {worker, state} = pop_in(state, [Access.key(:workers), worker_id]) @@ -503,6 +533,13 @@ defmodule Coflux.Orchestration.Server.Fleet do worker_id: worker_id, last_idle_at: activated_at || created_at, activated_at: activated_at, + # Targets live only in memory, so a session that reconnects + # after a restart has to declare them again - it is not ready + # until it does, and the deadline for doing so is armed by + # that reconnection rather than by the activation it did + # before the restart. + declared_at: nil, + ready_deadline_at: nil, activation_timeout: activation_timeout, reconnection_timeout: reconnection_timeout, total_executions: Map.get(assignment_counts_by_session, session_id, 0) diff --git a/server/lib/coflux/orchestration/server/rotation.ex b/server/lib/coflux/orchestration/server/rotation.ex index 591b34ae..fdd4bbcc 100644 --- a/server/lib/coflux/orchestration/server/rotation.ex +++ b/server/lib/coflux/orchestration/server/rotation.ex @@ -179,6 +179,13 @@ defmodule Coflux.Orchestration.Server.Rotation do {new_ws_id, new_ws_pools} end) + # Remap pool_failures: keyed by pool ID. A pool that no longer exists + # in the new epoch has nothing left to back off from. + pool_failures = + state.pool_failures + |> Enum.filter(fn {old_id, _} -> Map.has_key?(pool_map, old_id) end) + |> Map.new(fn {old_id, failures} -> {Map.fetch!(pool_map, old_id), failures} end) + # Remap workers: rekey map, update pool_id, workspace_id, session_id workers = Map.new(state.workers, fn {old_id, worker} -> @@ -262,6 +269,7 @@ defmodule Coflux.Orchestration.Server.Rotation do workspace_names: workspace_names, workspace_external_ids: workspace_external_ids, pools: pools, + pool_failures: pool_failures, workers: workers, worker_external_ids: worker_external_ids, sessions: sessions, diff --git a/server/lib/coflux/orchestration/server/routing.ex b/server/lib/coflux/orchestration/server/routing.ex index a18f9f18..a37cf75b 100644 --- a/server/lib/coflux/orchestration/server/routing.ex +++ b/server/lib/coflux/orchestration/server/routing.ex @@ -207,9 +207,17 @@ defmodule Coflux.Orchestration.Server.Routing do # --------------------------------------------------------------------------- # Sessions, pools and workers - def route(%SessionUpdated{} = e, _state), do: [{:sessions, e.workspace}] + # A pool-launched worker's connection state belongs to its pool as much + # as to the sessions list: it is the difference between a worker that is + # running and one that was merely started. + def route(%SessionUpdated{} = e, _state), + do: [{:sessions, e.workspace} | if(e.pool, do: [{:pool, e.workspace, e.pool}], else: [])] + def route(%SessionEnded{} = e, _state), do: [{:sessions, e.workspace}] - def route(%SessionConnected{} = e, _state), do: [{:sessions, e.workspace}] + + def route(%SessionConnected{} = e, state), + do: [{:sessions, e.workspace} | session_pool_topics(state, e.workspace, e.session)] + def route(%SessionExecuting{} = e, _state), do: [{:sessions, e.workspace}] def route(%SessionExecutions{} = e, _state) do @@ -249,4 +257,15 @@ defmodule Coflux.Orchestration.Server.Routing do :error -> [] end end + + defp session_pool_topics(state, workspace, session_external_id) do + with {:ok, session_id} <- Map.fetch(state.session_ids, session_external_id), + {:ok, session} <- Map.fetch(state.sessions, session_id), + worker_id when not is_nil(worker_id) <- session.worker_id, + {:ok, worker} <- Map.fetch(state.workers, worker_id) do + [{:pool, workspace, worker.pool_name}] + else + _ -> [] + end + end end diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index ea12c1dd..50820810 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -58,9 +58,35 @@ defmodule Coflux.Orchestration.Server.Scheduler do @connected_worker_poll_interval_ms 30_000 @disconnected_worker_poll_interval_ms 5_000 - # How long a worker sits idle before it is stopped. + # How long a *ready* worker sits idle before it is stopped. A worker + # that has never declared targets is not idle, it is still starting - + # see `Fleet.session_ready?/1`. @worker_idle_timeout_ms 5_000 + # How often to sweep while any worker exists, for the deadlines above. + @sweep_interval_ms 5_000 + + # The shortest gap between launches for one pool. + @pool_launch_interval_ms 10_000 + + # A pool whose launches keep failing backs off, so a bad image costs one + # attempt every few minutes rather than one every pass. Cleared when a + # worker from the pool becomes ready. + @pool_backoff_base_ms 5_000 + @pool_backoff_max_ms 300_000 + + # A launcher that cannot answer a poll is not evidence that the worker + # is gone, and giving up on a worker abandons whatever it is running - + # so a transient failure has to persist for this many consecutive polls + # *and* this long before the worker is deactivated. Erring late is + # cheap: the stale worker is polled again every pass and recovers as + # soon as the launcher does. + @poll_failure_threshold 3 + @poll_failure_grace_ms 300_000 + + # How long before asking the launcher again after a stop fails. + @stop_retry_interval_ms 30_000 + @default_activation_timeout_ms 600_000 @default_reconnection_timeout_ms 30_000 @@ -371,12 +397,12 @@ defmodule Coflux.Orchestration.Server.Scheduler do }) end) - state = + {state, next_launch_at} = if Enum.any?(unassigned) do # Track the most recent worker creation per pool, and which pools # already have a worker that isn't ready to accept work. We skip # launching for pools that have a worker still pending activation - # or that activated but hasn't registered any targets yet (e.g. + # or that activated but hasn't declared what it can run (e.g. # due to a misconfigured command or working directory). {latest_pool_launch_at, pools_with_pending_worker} = state.workers @@ -388,7 +414,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do pending = with session_id when not is_nil(session_id) <- worker.session_id, {:ok, session} <- Map.fetch(state.sessions, session_id), - false <- session.activated_at != nil and Enum.any?(session.targets) do + false <- Fleet.session_ready?(session) do MapSet.put(pending, worker.pool_id) else _ -> pending @@ -399,142 +425,187 @@ defmodule Coflux.Orchestration.Server.Scheduler do unassigned |> Enum.group_by(& &1.workspace_id) - |> Enum.reduce(state, fn {workspace_id, executions}, state -> - executions - |> Enum.map(fn execution -> - requires = - Fleet.effective_requires( - tag_sets, - execution.run_requires_tag_set_id, - execution.requires_tag_set_id - ) - - Fleet.choose_pool(state, execution, requires) - end) - |> Enum.reject(&is_nil/1) - |> Enum.uniq() - |> Enum.reject(&MapSet.member?(pools_with_pending_worker, &1)) - |> Enum.filter(&(now - Map.get(latest_pool_launch_at, &1, 0) > 10_000)) - |> Enum.reduce(state, fn pool_id, state -> - case Workers.create_worker(state.db, pool_id) do - {:ok, worker_id, worker_external_id, created_at} -> - {pool_name, pool} = - Enum.find( - Map.get(state.pools, workspace_id, %{}), - &(elem(&1, 1).id == pool_id) - ) + |> Enum.reduce({state, nil}, fn {workspace_id, executions}, {state, next_launch_at} -> + candidates = + executions + |> Enum.map(fn execution -> + requires = + Fleet.effective_requires( + tag_sets, + execution.run_requires_tag_set_id, + execution.requires_tag_set_id + ) - # Create a session for the pool-launched worker - activation_timeout = - Map.get(pool, :activation_timeout, @default_activation_timeout_ms) - - reconnection_timeout = - Map.get(pool, :reconnection_timeout, @default_reconnection_timeout_ms) - - pool_accepts = Map.get(pool, :accepts, %{}) - - session_opts = [ - provides: pool.provides, - accepts: pool_accepts, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout - ] - - {:ok, session_id, external_id, token, secret_hash, session_now} = - Sessions.create_session(state.db, workspace_id, worker_id, session_opts) - - session = %{ - external_id: external_id, - secret_hash: secret_hash, - connection: nil, - targets: %{}, - queue: [], - starting: MapSet.new(), - executing: MapSet.new(), - concurrency: 0, - draining: false, - workspace_id: workspace_id, - provides: pool.provides, - accepts: pool_accepts, - worker_id: worker_id, - last_idle_at: session_now, - activated_at: nil, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, - total_executions: 0 - } - - state - |> put_in([Access.key(:sessions), session_id], session) - |> put_in([Access.key(:session_ids), external_id], session_id) - |> Fleet.schedule_session_expiry(session_id, activation_timeout) - |> Listeners.maybe_schedule_idle_shutdown() - |> Fleet.call_launcher( - pool.launcher, - :launch, - [ - Fleet.build_launcher_env(state, workspace_id, token, pool.launcher), - pool.modules, + Fleet.choose_pool(state, execution, requires) + end) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> Enum.reject(&MapSet.member?(pools_with_pending_worker, &1)) + |> Map.new(&{&1, pool_launch_due_at(state, &1, latest_pool_launch_at)}) + + # A pool that wants a worker but can't have one yet is the only + # reason an otherwise-quiet project still needs waking: nothing + # else will happen until its throttle or backoff expires. + next_launch_at = + candidates + |> Map.values() + |> Enum.filter(&(&1 > now)) + |> Enum.min(fn -> nil end) + |> earliest(next_launch_at) + + state = + candidates + |> Enum.filter(fn {_pool_id, due_at} -> due_at <= now end) + |> Enum.map(&elem(&1, 0)) + |> Enum.reduce(state, fn pool_id, state -> + case Workers.create_worker(state.db, pool_id) do + {:ok, worker_id, worker_external_id, created_at} -> + {pool_name, pool} = + Enum.find( + Map.get(state.pools, workspace_id, %{}), + &(elem(&1, 1).id == pool_id) + ) + + # Create a session for the pool-launched worker + activation_timeout = + Map.get(pool, :activation_timeout, @default_activation_timeout_ms) + + reconnection_timeout = + Map.get(pool, :reconnection_timeout, @default_reconnection_timeout_ms) + + pool_accepts = Map.get(pool, :accepts, %{}) + + session_opts = [ + provides: pool.provides, + accepts: pool_accepts, + activation_timeout: activation_timeout, + reconnection_timeout: reconnection_timeout + ] + + {:ok, session_id, external_id, token, secret_hash, session_now} = + Sessions.create_session(state.db, workspace_id, worker_id, session_opts) + + session = %{ + external_id: external_id, + secret_hash: secret_hash, + connection: nil, + targets: %{}, + queue: [], + starting: MapSet.new(), + executing: MapSet.new(), + concurrency: 0, + draining: false, + workspace_id: workspace_id, + provides: pool.provides, + accepts: pool_accepts, + worker_id: worker_id, + last_idle_at: session_now, + activated_at: nil, + declared_at: nil, + ready_deadline_at: nil, + activation_timeout: activation_timeout, + reconnection_timeout: reconnection_timeout, + total_executions: 0 + } + + state + |> put_in([Access.key(:sessions), session_id], session) + |> put_in([Access.key(:session_ids), external_id], session_id) + |> Fleet.schedule_session_expiry(session_id, activation_timeout) + |> Listeners.maybe_schedule_idle_shutdown() + |> Fleet.call_launcher( pool.launcher, - %{pool_name: pool_name} - ], - fn state, result -> - {data, error} = - case result do - {:ok, {:ok, data}} -> {data, nil} - {:ok, {:error, error}} -> {nil, error} - :error -> {nil, "launch_crashed"} - end + :launch, + [ + Fleet.build_launcher_env(state, workspace_id, token, pool.launcher), + pool.modules, + pool.launcher, + %{pool_name: pool_name} + ], + fn state, result -> + {data, error} = + case result do + {:ok, {:ok, data}} -> {data, nil} + {:ok, {:error, error}} -> {nil, error} + :error -> {nil, "launch_crashed"} + end + + {:ok, started_at} = + Workers.create_worker_launch_result(state.db, worker_id, data, error) + + state = + Effects.emit(state, %WorkerLaunchResult{ + workspace: State.workspace_external_id(state, workspace_id), + pool: pool_name, + worker: worker_external_id, + started_at: started_at, + error: error + }) + + cond do + error -> + # Deactivating the worker pops it from state, and + # with it the only record that this pool was ever + # tried - so count the failure first, or the pool + # relaunches on the very next pass. + state + |> record_pool_launch_failure(pool_id) + |> Fleet.deactivate_worker(worker_id, error) + + Map.has_key?(state.workers, worker_id) -> + put_in( + state, + [Access.key(:workers), worker_id, Access.key(:data)], + data + ) - {:ok, started_at} = - Workers.create_worker_launch_result(state.db, worker_id, data, error) - - state = - state - |> put_in([Access.key(:workers), worker_id, Access.key(:data)], data) - |> Effects.emit(%WorkerLaunchResult{ - workspace: State.workspace_external_id(state, workspace_id), - pool: pool_name, - worker: worker_external_id, - started_at: started_at, - error: error - }) - - state = - if error do - Fleet.deactivate_worker(state, worker_id, error) - else - state + true -> + # The worker was deactivated while its launch was + # in flight. Nothing will ever connect to what was + # just started, and this result is the only thing + # that knows how to reach it, so stop it here + # rather than leaking it. + Fleet.call_launcher( + state, + pool.launcher, + :stop, + [data], + fn state, _result -> state end + ) end + end + ) + |> put_in([Access.key(:workers), worker_id], %{ + external_id: worker_external_id, + created_at: created_at, + pool_id: pool_id, + pool_name: pool_name, + workspace_id: workspace_id, + state: :active, + data: nil, + session_id: session_id, + stop_id: nil, + stop_retry_at: nil, + last_poll_at: nil, + polling: false, + poll_failures: 0, + first_poll_failure_at: nil + }) + |> put_in([Access.key(:worker_external_ids), worker_external_id], worker_id) + |> Effects.emit(%WorkerCreated{ + workspace: State.workspace_external_id(state, workspace_id), + pool: pool_name, + worker: worker_external_id, + created_at: created_at, + session: external_id + }) + end + end) - state - end - ) - |> put_in([Access.key(:workers), worker_id], %{ - external_id: worker_external_id, - created_at: created_at, - pool_id: pool_id, - pool_name: pool_name, - workspace_id: workspace_id, - state: :active, - data: nil, - session_id: session_id, - stop_id: nil, - last_poll_at: nil - }) - |> put_in([Access.key(:worker_external_ids), worker_external_id], worker_id) - |> Effects.emit(%WorkerCreated{ - workspace: State.workspace_external_id(state, workspace_id), - pool: pool_name, - worker: worker_external_id, - created_at: created_at, - session: external_id - }) - end - end) + {state, next_launch_at} end) else - state + {state, nil} end next_execute_after = @@ -544,67 +615,70 @@ defmodule Coflux.Orchestration.Server.Scheduler do state = state.workers - |> Enum.filter(fn {_worker_id, worker} -> - # TODO: don't poll if a poll is in progress? - if worker.data do - if is_nil(worker.last_poll_at) do - true - else - connection = - if worker.session_id && Map.has_key?(state.sessions, worker.session_id), - do: state.sessions[worker.session_id].connection - - interval_ms = - if connection, - do: @connected_worker_poll_interval_ms, - else: @disconnected_worker_poll_interval_ms - - now - worker.last_poll_at > interval_ms - end - else - false - end - end) + |> Enum.filter(fn {_worker_id, worker} -> poll_due?(state, worker, now) end) |> Enum.reduce(state, fn {worker_id, worker}, state -> {:ok, launcher} = Workspaces.get_launcher_for_pool(state.db, worker.pool_id) state |> Fleet.call_launcher(launcher, :poll, [worker.data], fn state, result -> + state = update_worker(state, worker_id, &%{&1 | polling: false}) + case result do {:ok, {:ok, true}} -> - state + clear_poll_failures(state, worker_id) {:ok, {:ok, false, error, logs}} -> + # The launcher knows the worker has gone, and this is the + # only place its exit code and log tail come from. Fleet.deactivate_worker(state, worker_id, error, logs) {:ok, {:error, _reason}} -> - Fleet.deactivate_worker(state, worker_id, "poll_error") + record_poll_failure(state, worker_id) :error -> - # TODO: ? - state + record_poll_failure(state, worker_id) end end) - |> put_in([Access.key(:workers), worker_id, :last_poll_at], now) + |> update_worker(worker_id, &%{&1 | last_poll_at: now, polling: true}) + end) + + # A worker that connected but never said what it can run is broken + # rather than idle - a bad command or working directory, typically - + # and the idle timeout below deliberately doesn't apply to it, so this + # is what bounds it. It counts against the pool for the same reason a + # failed launch does: without that, the pool relaunches immediately + # and repeats the whole thing. + unready = + Enum.filter(state.workers, fn {_worker_id, worker} -> + case worker_session(state, worker) do + {:ok, session} -> session.ready_deadline_at && now > session.ready_deadline_at + :error -> false + end end) + state = + Enum.reduce(unready, state, fn {worker_id, worker}, state -> + state + |> record_pool_launch_failure(worker.pool_id) + |> Fleet.deactivate_worker(worker_id, "no_targets") + end) + + # TODO: consider min/max pool size state = state.workers - |> Enum.group_by(fn {_, worker} -> worker.pool_name end) - |> Enum.flat_map(fn {_pool_name, workers} -> - # TODO: consider min/max pool size - Enum.filter(workers, fn {_worker_id, worker} -> - # TODO: better way to check launched than checking existence of data? - if worker.state == :active && worker.session_id && worker.data do - session = Map.fetch!(state.sessions, worker.session_id) - idle_time = now - session.last_idle_at - - if Enum.empty?(session.starting) && Enum.empty?(session.executing) && - idle_time >= @worker_idle_timeout_ms do - true - end - end - end) + |> Enum.filter(fn {_worker_id, worker} -> + # TODO: better way to check launched than checking existence of data? + with true <- worker.state == :active && !is_nil(worker.data), + {:ok, session} <- worker_session(state, worker), + # Only a worker that has been in a position to take work can + # be surplus to it. One that is still starting has an idle + # time, but it means nothing. + true <- Fleet.session_ready?(session) do + Enum.empty?(session.starting) && Enum.empty?(session.executing) && + now - session.last_idle_at >= @worker_idle_timeout_ms + else + _ -> false + end end) |> Enum.reduce(state, fn {worker_id, worker}, state -> Fleet.update_worker_state( @@ -618,23 +692,14 @@ defmodule Coflux.Orchestration.Server.Scheduler do state = state.workers - |> Enum.filter(fn {_worker_id, worker} -> - if worker.session_id do - if worker.state == :draining && worker.data && !worker.stop_id do - session = Map.fetch!(state.sessions, worker.session_id) - Enum.empty?(session.starting) && Enum.empty?(session.executing) - end - else - !is_nil(worker.data) - end - end) + |> Enum.filter(fn {_worker_id, worker} -> stop_due?(state, worker, now) end) |> Enum.reduce(state, fn {worker_id, worker}, state -> {:ok, worker_stop_id, stopping_at} = Workers.create_worker_stop(state.db, worker_id) {:ok, launcher} = Workspaces.get_launcher_for_pool(state.db, worker.pool_id) state = state - |> put_in([Access.key(:workers), worker_id, :stop_id], worker_stop_id) + |> update_worker(worker_id, &%{&1 | stop_id: worker_stop_id, stop_retry_at: nil}) |> Effects.emit(%WorkerStopping{ workspace: State.workspace_external_id(state, worker.workspace_id), pool: worker.pool_name, @@ -656,45 +721,40 @@ defmodule Coflux.Orchestration.Server.Scheduler do error: nil }) - {:ok, {:error, _reason}} -> - # Stop failed (e.g. connection refused) — treat as stopped - {:ok, stopped_at} = - Workers.create_worker_stop_result(state.db, worker_stop_id, nil) - - Effects.emit(state, %WorkerStopResult{ - workspace: State.workspace_external_id(state, worker.workspace_id), - pool: worker.pool_name, - worker: worker.external_id, - stopped_at: stopped_at, - error: nil - }) + {:ok, {:error, reason}} -> + record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) :error -> - # TODO: get error details - error = %{} - - {:ok, _} = - Workers.create_worker_stop_result(state.db, worker_stop_id, error) - - state = - Effects.emit(state, %WorkerStopResult{ - workspace: State.workspace_external_id(state, worker.workspace_id), - pool: worker.pool_name, - worker: worker.external_id, - stopped_at: nil, - error: error - }) - - # TODO: unset 'stop_id' of worker in state? (so it can be retried? but somehow limit rate?) - state + record_stop_failure(state, worker_id, worker, worker_stop_id, "stop_crashed") end end) end) + # While any worker exists there are deadlines to sweep for - polls, + # idle drains, readiness, stop retries - so keep the periodic pass. + # With no workers the only thing still on a clock is a pool waiting + # out its throttle or backoff; with neither, nothing here will change + # until something happens, and everything that happens asks for a + # pass of its own. + sweep_delay_ms = + cond do + map_size(state.workers) > 0 -> + @sweep_interval_ms + + # Deactivating a worker above changed what the launch pass, which + # has already run, would have decided - so run another, now that + # its pool is no longer waiting on a worker that is never coming. + unready != [] -> + 0 + + true -> + next_launch_at && max(0, next_launch_at - now) + end + delay_ms = [ if(next_execute_after, do: trunc(next_execute_after) - System.os_time(:millisecond)), - if(state.workers, do: 5_000) + sweep_delay_ms ] |> Enum.reject(&is_nil/1) |> Enum.min(fn -> nil end) @@ -714,4 +774,157 @@ defmodule Coflux.Orchestration.Server.Scheduler do state end + + # Applies `fun` to a worker, or does nothing if it has been deactivated + # since - which a launcher callback landing late always might find. + defp update_worker(state, worker_id, fun) do + if Map.has_key?(state.workers, worker_id) do + update_in(state, [Access.key(:workers), worker_id], fun) + else + state + end + end + + defp worker_session(state, worker) do + if worker.session_id, do: Map.fetch(state.sessions, worker.session_id), else: :error + end + + # When a pool may next launch: never sooner than the launch interval + # after its last worker, and not until it has served out any backoff + # from launches that failed. + defp pool_launch_due_at(state, pool_id, latest_pool_launch_at) do + throttled_until = Map.get(latest_pool_launch_at, pool_id, 0) + @pool_launch_interval_ms + + case Map.get(state.pool_failures, pool_id) do + nil -> + throttled_until + + %{failures: failures, last_attempt_at: last_attempt_at} -> + max(throttled_until, last_attempt_at + pool_backoff_ms(failures)) + end + end + + defp pool_backoff_ms(failures) do + min(@pool_backoff_max_ms, @pool_backoff_base_ms * Integer.pow(2, min(failures - 1, 16))) + end + + defp record_pool_launch_failure(state, pool_id) do + update_in( + state, + [Access.key(:pool_failures), Access.key(pool_id, %{failures: 0, last_attempt_at: 0})], + fn %{failures: failures} -> + %{failures: failures + 1, last_attempt_at: System.os_time(:millisecond)} + end + ) + end + + defp poll_due?(state, worker, now) do + cond do + is_nil(worker.data) -> + false + + worker.polling -> + false + + is_nil(worker.last_poll_at) -> + true + + true -> + connected = + case worker_session(state, worker) do + {:ok, session} -> !is_nil(session.connection) + :error -> false + end + + interval_ms = + if connected, + do: @connected_worker_poll_interval_ms, + else: @disconnected_worker_poll_interval_ms + + now - worker.last_poll_at > interval_ms + end + end + + defp clear_poll_failures(state, worker_id) do + update_worker(state, worker_id, &%{&1 | poll_failures: 0, first_poll_failure_at: nil}) + end + + # A launcher that couldn't answer says nothing about the worker, so this + # only gives up once the failures have persisted for long enough to rule + # out the launcher itself being briefly unavailable. Both bounds matter: + # a disconnected worker is polled every few seconds, so a count alone + # would expire inside an ordinary Docker daemon restart. + defp record_poll_failure(state, worker_id) do + case Map.fetch(state.workers, worker_id) do + :error -> + state + + {:ok, worker} -> + now = System.os_time(:millisecond) + failures = worker.poll_failures + 1 + first_failure_at = worker.first_poll_failure_at || now + + if failures >= @poll_failure_threshold and + now - first_failure_at > @poll_failure_grace_ms do + Fleet.deactivate_worker(state, worker_id, "poll_error") + else + update_worker( + state, + worker_id, + &%{&1 | poll_failures: failures, first_poll_failure_at: first_failure_at} + ) + end + end + end + + defp stop_due?(state, worker, now) do + cond do + # Nothing to ask the launcher about until the launch has landed. + is_nil(worker.data) -> false + # A stop is already in flight, or has already succeeded. + worker.stop_id -> false + # A previous stop failed; wait before asking again. + worker.stop_retry_at && now < worker.stop_retry_at -> false + # The session has gone, so there is nothing left to drain. + is_nil(worker.session_id) -> true + worker.state != :draining -> false + true -> drained?(state, worker) + end + end + + defp drained?(state, worker) do + case worker_session(state, worker) do + {:ok, session} -> Enum.empty?(session.starting) && Enum.empty?(session.executing) + :error -> true + end + end + + # A stop that failed is recorded as one: the container may well still be + # running, and reporting it as stopped both misleads whoever is watching + # and means nothing ever tries again. Clearing `stop_id` is what allows + # the retry; `stop_retry_at` is what keeps it from being immediate. + defp record_stop_failure(state, worker_id, worker, worker_stop_id, error) do + {:ok, _} = Workers.create_worker_stop_result(state.db, worker_stop_id, error) + + state + |> Effects.emit(%WorkerStopResult{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + stopped_at: nil, + error: error + }) + |> update_worker( + worker_id, + &%{&1 | stop_id: nil, stop_retry_at: System.os_time(:millisecond) + @stop_retry_interval_ms} + ) + end + + defp earliest(nil, b), do: b + defp earliest(a, nil), do: a + defp earliest(a, b), do: min(a, b) + + defp to_error(reason) when is_binary(reason), do: reason + defp to_error(reason) when is_atom(reason), do: Atom.to_string(reason) + defp to_error(reason), do: inspect(reason) end diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index 079e8015..bc06ee28 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -44,6 +44,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do ResultRecorded, RunCreated, RunOutcome, + SessionConnected, SessionExecutions, StepArguments, StepCreated, @@ -507,7 +508,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do with {:ok, workspace_id} <- Permissions.resolve_workspace_external_id(state, workspace_external_id) do pool = state.pools |> Map.get(workspace_id, %{}) |> Map.get(pool_name) - {:ok, pool_workers} = Workers.get_pool_workers(state.db, pool_name) + {:ok, pool_workers} = Workers.get_pool_workers(state.db, workspace_id, pool_name) if is_nil(pool) and pool_workers == [] do {:error, :not_found} @@ -525,14 +526,18 @@ defmodule Coflux.Orchestration.Server.Snapshots do deactivated_at, error, logs, total_executions} -> worker = Map.get(state.workers, worker_id) - session_external_id = + session = if worker && worker.session_id do case Map.fetch(state.sessions, worker.session_id) do - {:ok, session} -> session.external_id + {:ok, session} -> session :error -> nil end end + session_external_id = session && session.external_id + + # So a subscriber joining now sees the same connection state + # a subscriber watching all along would have. [ %WorkerCreated{ workspace: workspace_external_id, @@ -604,7 +609,17 @@ defmodule Coflux.Orchestration.Server.Snapshots do pool: pool_name, executions: total_executions } - ] + ] ++ + if(session, + do: [ + %SessionConnected{ + workspace: workspace_external_id, + session: session.external_id, + connected: !is_nil(session.connection) + } + ], + else: [] + ) end) {:ok, definition ++ workers} diff --git a/server/lib/coflux/orchestration/server/state.ex b/server/lib/coflux/orchestration/server/state.ex index ae2ae4cc..34f1a29e 100644 --- a/server/lib/coflux/orchestration/server/state.ex +++ b/server/lib/coflux/orchestration/server/state.ex @@ -84,13 +84,20 @@ defmodule Coflux.Orchestration.Server.State do # named on the wire. workspace_names: %{}, - # worker_id -> %{created_at, pool_id, pool_name, workspace_id, state, data, session_id, stop_id, last_poll_at} + # worker_id -> %{created_at, pool_id, pool_name, workspace_id, state, data, session_id, stop_id, stop_retry_at, last_poll_at, polling, poll_failures, first_poll_failure_at} # # Workers this server launched and hasn't deactivated. # `session_id` links to the session that connected for it, if # one has. Rebuilt at boot by `Fleet.load/1`, which deactivates # any that were launched but never got a session - nothing is # going to connect to those. + # + # `polling` is set while a poll is in flight, so a slow + # launcher can't accumulate overlapping polls. The poll failure + # counters track *transient* launcher errors only: a launcher + # that cannot answer is not evidence the worker is gone, so a + # worker is only given up on once both counters pass their + # thresholds (see `Scheduler`). workers: %{}, # ref -> {pid, session_id} @@ -100,7 +107,7 @@ defmodule Coflux.Orchestration.Server.State do # why a session is separate from it and outlives it. connections: %{}, - # session_id -> %{external_id, connection, targets, queue, starting, executing, concurrency, workspace_id, provides, accepts, worker_id, last_idle_at, activated_at, activation_timeout, reconnection_timeout} + # session_id -> %{external_id, connection, targets, queue, starting, executing, concurrency, workspace_id, provides, accepts, worker_id, last_idle_at, activated_at, declared_at, activation_timeout, reconnection_timeout} # # One entry per worker session. A session outlives its # connection - a worker that drops reconnects into the same one @@ -109,6 +116,12 @@ defmodule Coflux.Orchestration.Server.State do # meanwhile. Restored at boot by `Fleet.load/1` (with an empty # queue: commands buffered for a session do not survive a # restart); owned by `Fleet` thereafter. + # + # `activated_at` is when the worker first connected, + # `declared_at` when it first said what it can run. Both are + # needed before the session counts as ready: until then it has + # never been able to take work, so its idleness means nothing + # (see `Fleet.session_ready?/1`). sessions: %{}, # external_id -> session_id @@ -202,6 +215,17 @@ defmodule Coflux.Orchestration.Server.State do # any) is deactivated at boot for having no session. launcher_tasks: %{}, + # pool_id -> %{failures, last_attempt_at} + # + # Consecutive failed launches per pool, and when the last one + # was attempted, so a pool whose launches keep failing backs + # off instead of retrying every pass. Keyed by pool id rather + # than name: pool rows are immutable, so editing a pool mints a + # new id and the backoff clears itself. Cleared when a worker + # from the pool becomes ready. In-memory only - a restart is + # worth one free attempt. + pool_failures: %{}, + # `Coflux.Store.Index`: the Bloom filter per archived epoch, # so a lookup for a row that cannot be in an epoch never opens # it. Persisted, unlike everything else here. diff --git a/server/lib/coflux/orchestration/workers.ex b/server/lib/coflux/orchestration/workers.ex index f56a76a7..eafb4434 100644 --- a/server/lib/coflux/orchestration/workers.ex +++ b/server/lib/coflux/orchestration/workers.ex @@ -164,37 +164,55 @@ defmodule Coflux.Orchestration.Workers do end end - def get_pool_workers(db, pool_name, limit \\ 100) do - # TODO: decode errors? - query( - db, - """ - SELECT w.id, w.external_id, w.created_at, r.created_at, r.error, s.created_at, sr.created_at, sr.error, - d.created_at, d.error, wl.content, - (SELECT COUNT(*) FROM assignments AS a - INNER JOIN sessions AS ses ON ses.id = a.session_id - WHERE ses.worker_id = w.id) AS total_executions - FROM workers AS w - INNER JOIN pools AS p ON p.id = w.pool_id - LEFT JOIN worker_launch_results AS r ON r.worker_id = w.id - LEFT JOIN worker_stops AS s ON s.id = ( - SELECT id - FROM worker_stops - WHERE worker_id = w.id - ORDER BY created_at DESC - LIMIT 1 - ) - LEFT JOIN worker_stop_results AS sr ON sr.worker_stop_id = s.id - LEFT JOIN worker_deactivations AS d ON d.worker_id = w.id - LEFT JOIN worker_logs AS wl ON wl.id = d.worker_log_id - WHERE p.name = ?1 - ORDER BY w.created_at DESC - LIMIT ?2 - """, - {pool_name, limit} - ) + # Pool names are unique per workspace, not per project, so the workspace + # is part of identifying a pool - without it this returns the workers of + # every same-named pool in the project. + def get_pool_workers(db, workspace_id, pool_name, limit \\ 100) do + case query( + db, + """ + SELECT w.id, w.external_id, w.created_at, r.created_at, r.error, s.created_at, sr.created_at, sr.error, + d.created_at, d.error, wl.content, + (SELECT COUNT(*) FROM assignments AS a + INNER JOIN sessions AS ses ON ses.id = a.session_id + WHERE ses.worker_id = w.id) AS total_executions + FROM workers AS w + INNER JOIN pools AS p ON p.id = w.pool_id + LEFT JOIN worker_launch_results AS r ON r.worker_id = w.id + LEFT JOIN worker_stops AS s ON s.id = ( + SELECT id + FROM worker_stops + WHERE worker_id = w.id + ORDER BY created_at DESC + LIMIT 1 + ) + LEFT JOIN worker_stop_results AS sr ON sr.worker_stop_id = s.id + LEFT JOIN worker_deactivations AS d ON d.worker_id = w.id + LEFT JOIN worker_logs AS wl ON wl.id = d.worker_log_id + WHERE p.workspace_id = ?1 AND p.name = ?2 + ORDER BY w.created_at DESC + LIMIT ?3 + """, + {workspace_id, pool_name, limit} + ) do + {:ok, rows} -> + {:ok, + Enum.map(rows, fn {worker_id, external_id, created_at, started_at, start_error, + stopping_at, stopped_at, stop_error, deactivated_at, error, logs, + total_executions} -> + # Launch and stop errors are stored encoded; deactivation errors + # are not. Decode here so a worker read back from the database + # carries the same shapes as one observed live. + {worker_id, external_id, created_at, started_at, decode_error(start_error), + stopping_at, stopped_at, decode_error(stop_error), deactivated_at, error, logs, + total_executions} + end)} + end end + defp decode_error(nil), do: nil + defp decode_error(value), do: Jason.decode!(value) + defp generate_uuid() do <> = :crypto.strong_rand_bytes(16) diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index 6a1bbb7f..794bc558 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -496,17 +496,24 @@ defmodule Coflux.Orchestration.Workspaces do pool_definition_id end + # Definitions are content-addressed, so a patch that re-states + # what is already there resolves to the same definition and + # writes no row. Say so: the caller drains the pool's workers + # on a change, and re-applying the current configuration + # should not restart anything. if pool_definition_id != existing_pool_definition_id do - insert_workspace_pool( - db, - workspace_id, - pool_name, - pool_definition_id, - now, - created_by - ) + case insert_workspace_pool( + db, + workspace_id, + pool_name, + pool_definition_id, + now, + created_by + ) do + {:ok, pool_id} -> {:ok, pool_id, :updated} + end else - {:ok, existing_pool_id} + {:ok, existing_pool_id, :unchanged} end end end diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index f7c1ea97..82021763 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -54,17 +54,29 @@ defmodule Coflux.Topics.Pool do :docker -> %{type: "docker", image: launcher.image} |> maybe_put(:dockerHost, Map.get(launcher, :docker_host)) + |> maybe_put(:networkMode, Map.get(launcher, :network_mode)) :process -> %{type: "process", directory: launcher.directory} + # `token` is deliberately absent: this shape is delivered to every + # subscriber of the topic, and the launcher's credentials are not + # part of what a pool looks like. `caCert` is a path on the + # server's host rather than a credential, so it stays. Everything + # else a pool is configured with belongs here too, or `pools get` + # shows less than `pools export` does. :kubernetes -> %{type: "kubernetes", image: launcher.image} |> maybe_put(:namespace, Map.get(launcher, :namespace)) |> maybe_put(:apiServer, Map.get(launcher, :api_server)) |> maybe_put(:serviceAccount, Map.get(launcher, :service_account)) + |> maybe_put(:caCert, Map.get(launcher, :ca_cert)) |> maybe_put(:insecure, Map.get(launcher, :insecure)) |> maybe_put(:imagePullPolicy, Map.get(launcher, :image_pull_policy)) + |> maybe_put(:nodeSelector, Map.get(launcher, :node_selector)) + |> maybe_put(:tolerations, Map.get(launcher, :tolerations)) + |> maybe_put(:imagePullSecrets, Map.get(launcher, :image_pull_secrets)) + |> maybe_put(:hostAliases, Map.get(launcher, :host_aliases)) |> maybe_put(:labels, Map.get(launcher, :labels)) |> maybe_put(:annotations, Map.get(launcher, :annotations)) |> maybe_put(:activeDeadlineSeconds, Map.get(launcher, :active_deadline_seconds)) @@ -93,7 +105,9 @@ defmodule Coflux.Topics.Pool.Model do alias Coflux.Events.{ PoolStateChanged, PoolUpdated, + SessionConnected, SessionExecutions, + SessionUpdated, WorkerCreated, WorkerDeactivated, WorkerLaunchResult, @@ -128,6 +142,7 @@ defmodule Coflux.Topics.Pool.Model do logs: nil, state: :active, session: e.session, + connected: false, executions: 0 } @@ -159,6 +174,15 @@ defmodule Coflux.Topics.Pool.Model do def apply(model, %SessionExecutions{} = e), do: update(model, e.worker, &%{&1 | executions: e.executions}) + # Session events name a session, not a worker, so they land on whichever + # worker was launched with it - and on none, for a session that isn't a + # pool worker's. + def apply(model, %SessionUpdated{} = e), + do: update_by_session(model, e.session, &%{&1 | connected: e.connected}) + + def apply(model, %SessionConnected{} = e), + do: update_by_session(model, e.session, &%{&1 | connected: e.connected}) + defp update(model, worker, fun) do case Map.fetch(model.workers, worker) do {:ok, entry} -> %{model | workers: Map.put(model.workers, worker, fun.(entry))} @@ -166,6 +190,13 @@ defmodule Coflux.Topics.Pool.Model do end end + defp update_by_session(model, session, fun) do + case Enum.find(model.workers, fn {_id, worker} -> worker.session == session end) do + {worker_id, _} -> update(model, worker_id, fun) + nil -> model + end + end + def project(model) do %{ pool: Pool.build_pool(model.pool), @@ -184,6 +215,7 @@ defmodule Coflux.Topics.Pool.Model do logs: worker.logs, state: worker.state, sessionId: worker.session, + connected: worker.connected, executions: worker.executions }} end) diff --git a/tests/support/adapter.py b/tests/support/adapter.py index 77cffa44..4f1e1d91 100644 --- a/tests/support/adapter.py +++ b/tests/support/adapter.py @@ -9,9 +9,16 @@ import socket import sys import threading +import time def discover(args): + # --discover-delay stands in for a worker that is slow to start: a big + # import, or an image still being pulled. The worker has a session + # from the moment the server launches it, but can't say what it runs + # until this returns. + if args.discover_delay: + time.sleep(args.discover_delay) with open(args.manifest) as f: print(f.read(), end="") @@ -67,6 +74,7 @@ def socket_to_stdout(): parser = argparse.ArgumentParser() parser.add_argument("--manifest", required=True) parser.add_argument("--socket") + parser.add_argument("--discover-delay", type=float, default=0) parser.add_argument("command", choices=["discover", "execute"]) parser.add_argument("modules", nargs="*") args = parser.parse_args() diff --git a/tests/support/cli.py b/tests/support/cli.py index 75e5b1e8..d9de68dc 100644 --- a/tests/support/cli.py +++ b/tests/support/cli.py @@ -366,6 +366,34 @@ def pools_launches(name, host=None, workspace="default"): return json.loads(result.stdout) +def pools_enable(name, host=None, workspace="default"): + _coflux("pools", "enable", name, host=host, workspace=workspace, output=None) + + +def pools_disable(name, host=None, workspace="default"): + _coflux("pools", "disable", name, host=host, workspace=workspace, output=None) + + +def pools_export(include_secrets=False, host=None, workspace="default"): + args = ["pools", "export"] + if include_secrets: + args.append("--include-secrets") + result = _coflux(*args, host=host, workspace=workspace, output=None) + return result.stdout + + +def pools_import(path, host=None, workspace="default"): + _coflux( + "pools", + "import", + str(path), + "--yes", + host=host, + workspace=workspace, + output=None, + ) + + def worker( modules, adapter, diff --git a/tests/support/server.py b/tests/support/server.py index 3f52db20..167b6e7a 100644 --- a/tests/support/server.py +++ b/tests/support/server.py @@ -82,7 +82,9 @@ def _start_local(self, timeout): "COFLUX_DATA_DIR": self.data_dir, "COFLUX_PUBLIC_HOST": "%.localhost:" + str(self.port), "COFLUX_REQUIRE_AUTH": "false", - "COFLUX_LAUNCHER_TYPES": "process,docker", + # kubernetes is allowed so that its configuration can be + # exercised; storing a pool config contacts no cluster. + "COFLUX_LAUNCHER_TYPES": "process,docker,kubernetes", "COFLUX_SUPER_TOKEN_HASH": hashlib.sha256(SUPER_TOKEN.encode()).hexdigest(), "COFLUX_CLI_PATH": cli_path, **self._extra_env, diff --git a/tests/test_pools.py b/tests/test_pools.py index 13919d94..5729fc7e 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -408,3 +408,192 @@ def test_multiple_modules(self, pool_env): result_b = poll_result(resp_b["runId"], host, timeout=_RESULT_TIMEOUT) assert result_b["value"]["data"] == "from_b" + + +class TestWorkerReadiness: + def test_slow_starting_worker_is_not_drained(self, pool_env): + """A worker slower to start than the idle timeout still gets work. + + The server creates the session when it launches the worker, but the + worker can't accept anything until it has connected and declared + its targets. Anything that measures idleness from before that point + drains the worker while it is still starting, and it is stopped + having never run a thing. + """ + host = pool_env["host"] + executor = pool_env["executor"] + manifest_path = pool_env["manifest_path"] + socket_path = pool_env["socket_path"] + + with open(manifest_path, "w") as f: + json.dump(manifest([workflow("test", "my_workflow")]), f) + + base_adapter = [ + "python3", + ADAPTER_SCRIPT, + "--manifest", + manifest_path, + "--socket", + socket_path, + ] + + # Comfortably longer than the server's 5s idle timeout. + slow_adapter = base_adapter + ["--discover-delay", "8"] + + cli.pools_create( + "test-pool", + type="process", + modules=["test"], + process_dir=str(pool_env["worker_dir"]), + adapter=slow_adapter, + host=host, + ) + + # Registered with the prompt adapter: only the launched worker + # should be slow. + cli.manifests_register("test", adapter=",".join(base_adapter), host=host) + + resp = cli.submit("test/my_workflow", host=host) + + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="ok") + + result = poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + assert result["value"]["data"] == "ok" + + +class TestPoolState: + def test_enable_unknown_pool_is_rejected(self, pool_env): + """Enabling a pool that doesn't exist fails, and changes nothing. + + The name is not a pool, so there is nothing to enable. Recording + the state anyway would leave behind an entry that looks like a pool + but has no launcher and no modules, which the scheduler then trips + over on its next pass. + """ + host = pool_env["host"] + targets = [workflow("test", "my_workflow")] + _setup_pool(pool_env, targets) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.pools_enable("no-such-pool", host=host) + assert "not_found" in exc_info.value.stderr + + assert "no-such-pool" not in cli.pools_list(host=host) + + # The next pass still runs: a phantom pool would crash it. + executor = pool_env["executor"] + resp = cli.submit("test/my_workflow", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="ok") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + def test_disable_unknown_pool_is_rejected(self, pool_env): + """Disabling a pool that doesn't exist fails, and changes nothing.""" + host = pool_env["host"] + _setup_pool(pool_env, [workflow("test", "my_workflow")]) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.pools_disable("no-such-pool", host=host) + assert "not_found" in exc_info.value.stderr + + assert "no-such-pool" not in cli.pools_list(host=host) + + def test_disable_and_enable_round_trip(self, pool_env): + """A real pool can be disabled and enabled again.""" + host = pool_env["host"] + _setup_pool(pool_env, [workflow("test", "my_workflow")]) + + cli.pools_disable("test-pool", host=host) + assert cli.pools_get("test-pool", host=host)["state"] == "disabled" + + cli.pools_enable("test-pool", host=host) + assert cli.pools_get("test-pool", host=host)["state"] == "active" + + +class TestPoolModules: + def test_wildcard_modules_are_rejected(self, pool_env): + """A pool's modules are names, not patterns. + + The same list is handed to the launcher as the worker's arguments, + so a wildcard would be passed to the worker to import as well as + matching no execution - a pool that silently never runs anything. + """ + host = pool_env["host"] + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.pools_create( + "wildcard-pool", + type="process", + modules=["myapp.*"], + process_dir=str(pool_env["worker_dir"]), + host=host, + ) + assert "bad_request" in exc_info.value.stderr + + assert "wildcard-pool" not in cli.pools_list(host=host) + + +class TestPoolSecrets: + def _create_kubernetes_pool(self, host, name="k8s-pool"): + cli._coflux( + "pools", + "create", + name, + "--type", + "kubernetes", + "--set", + "image=myorg/worker:latest", + "--set", + "token=super-secret-token", + "--set", + "apiServer=https://k8s.example.com", + "--modules", + "test", + host=host, + output=None, + ) + + def test_export_redacts_secrets_by_default(self, pool_env): + """An export doesn't put the cluster token on disk unasked.""" + host = pool_env["host"] + self._create_kubernetes_pool(host) + + exported = cli.pools_export(host=host) + assert "super-secret-token" not in exported + assert "" in exported + # Everything that isn't a secret is still there. + assert "https://k8s.example.com" in exported + + def test_export_with_secrets_round_trips(self, pool_env, tmp_path): + """--include-secrets gives a config that imports back unchanged.""" + host = pool_env["host"] + self._create_kubernetes_pool(host) + + exported = cli.pools_export(include_secrets=True, host=host) + assert "super-secret-token" in exported + + path = tmp_path / "pools.toml" + path.write_text(exported) + cli.pools_import(path, host=host) + + again = cli.pools_export(include_secrets=True, host=host) + assert "super-secret-token" in again + + def test_importing_a_redacted_export_is_refused(self, pool_env, tmp_path): + """A redacted export can't silently clear the secrets it omits.""" + host = pool_env["host"] + self._create_kubernetes_pool(host) + + path = tmp_path / "pools.toml" + path.write_text(cli.pools_export(host=host)) + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.pools_import(path, host=host) + assert "k8s-pool" in exc_info.value.stderr + assert "--include-secrets" in exc_info.value.stderr + + # And the real token is untouched. + assert "super-secret-token" in cli.pools_export(include_secrets=True, host=host) From 5154f886c680468353ec48d66beefd1a7f4c4f7e Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 19:58:51 +0100 Subject: [PATCH 02/21] Add initial ECS support --- cli/CHANGELOG.md | 4 +- cli/cmd/coflux/pools.go | 64 ++- docs/docs/cli_reference.md | 4 +- docs/docs/pools.md | 65 ++- docs/docs/server_config.md | 4 +- server/lib/coflux/config.ex | 4 +- server/lib/coflux/handlers/api.ex | 192 ++++++- .../lib/coflux/launchers/aws_credentials.ex | 171 +++++++ server/lib/coflux/launchers/ecs.ex | 470 ++++++++++++++++++ .../lib/coflux/orchestration/server/fleet.ex | 1 + .../coflux/orchestration/server/scheduler.ex | 14 +- server/lib/coflux/orchestration/workspaces.ex | 2 + server/lib/coflux/topics/pool.ex | 20 + tests/support/ecs.py | 236 +++++++++ tests/support/server.py | 5 +- tests/test_pools.py | 301 +++++++++++ 16 files changed, 1526 insertions(+), 31 deletions(-) create mode 100644 server/lib/coflux/launchers/aws_credentials.ex create mode 100644 server/lib/coflux/launchers/ecs.ex create mode 100644 tests/support/ecs.py diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 6c79a9c8..34ad6bee 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.13.0 -No changes. +Enhancements: + +- Adds `--type ecs` support for `pools create` and `pools update`. ## 0.12.0 diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 7d6a28d0..0ef6c4c4 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -186,6 +186,41 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if policy := getString(launcher, "imagePullPolicy"); policy != "" { fmt.Printf("Image pull policy: %s\n", policy) } + if cluster := getString(launcher, "cluster"); cluster != "" { + fmt.Printf("Cluster: %s\n", cluster) + } + if taskDef := getString(launcher, "taskDefinition"); taskDef != "" { + fmt.Printf("Task definition: %s\n", taskDef) + } + if region := getString(launcher, "region"); region != "" { + fmt.Printf("Region: %s\n", region) + } + if container := getString(launcher, "containerName"); container != "" { + fmt.Printf("Container: %s\n", container) + } + if provider := getString(launcher, "capacityProvider"); provider != "" { + fmt.Printf("Capacity provider: %s\n", provider) + } else if launchType := getString(launcher, "launchType"); launchType != "" { + fmt.Printf("Launch type: %s\n", launchType) + } + if subnets := getStringSlice(launcher, "subnets"); len(subnets) > 0 { + fmt.Printf("Subnets: %s\n", strings.Join(subnets, ", ")) + } + if groups := getStringSlice(launcher, "securityGroups"); len(groups) > 0 { + fmt.Printf("Security groups: %s\n", strings.Join(groups, ", ")) + } + if publicIP, ok := launcher["assignPublicIp"].(bool); ok && publicIP { + fmt.Printf("Public IP: yes\n") + } + if version := getString(launcher, "platformVersion"); version != "" { + fmt.Printf("Platform version: %s\n", version) + } + if keyID := getString(launcher, "accessKeyId"); keyID != "" { + fmt.Printf("Access key ID: %s\n", keyID) + } + if endpoint := getString(launcher, "endpoint"); endpoint != "" { + fmt.Printf("Endpoint: %s\n", endpoint) + } printServerHost(launcher) if adapter := getStringSlice(launcher, "adapter"); len(adapter) > 0 { fmt.Printf("Adapter: %s\n", strings.Join(adapter, " ")) @@ -514,14 +549,15 @@ var poolsCreateCmd = &cobra.Command{ Short: "Create a pool", Long: `Create a new pool with the specified launcher type and configuration. -Use --type to specify the launcher type (kubernetes, docker, or process). +Use --type to specify the launcher type (kubernetes, docker, process, or ecs). Use --set to set field values. Values are parsed as JSON if valid, otherwise treated as strings. Examples: coflux pools create my-pool --type kubernetes --set image=foo:latest --set namespace=default coflux pools create my-pool --type docker --set image=myapp:v1 --modules mod1,mod2 - coflux pools create my-pool --type process --set directory=/app --set concurrency=5`, + coflux pools create my-pool --type process --set directory=/app --set concurrency=5 + coflux pools create my-pool --type ecs --set cluster=workers --set taskDefinition=myapp --set region=eu-west-2 --set subnets=subnet-1`, Args: cobra.ExactArgs(1), RunE: runPoolsCreate, } @@ -548,7 +584,7 @@ Examples: func init() { // pools create flags - poolsCreateCmd.Flags().String("type", "", "Launcher type (kubernetes, docker, process)") + poolsCreateCmd.Flags().String("type", "", "Launcher type (kubernetes, docker, process, ecs)") _ = poolsCreateCmd.MarkFlagRequired("type") poolsCreateCmd.Flags().StringArray("set", nil, "Set a field value (key=value)") poolsCreateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted") @@ -584,13 +620,18 @@ var poolTopLevelFields = map[string]bool{ // launcherFields lists valid launcher field names. var launcherFields = map[string]bool{ - "image": true, "dockerHost": true, "directory": true, + "image": true, "dockerHost": true, "networkMode": true, "directory": true, "namespace": true, "serviceAccount": true, "apiServer": true, "token": true, "caCert": true, "insecure": true, "imagePullPolicy": true, "nodeSelector": true, "tolerations": true, "imagePullSecrets": true, "hostAliases": true, "resources": true, "labels": true, "annotations": true, "activeDeadlineSeconds": true, "volumes": true, "volumeMounts": true, + "cluster": true, "taskDefinition": true, "region": true, + "containerName": true, "launchType": true, "capacityProvider": true, + "subnets": true, "securityGroups": true, "assignPublicIp": true, + "platformVersion": true, "accessKeyId": true, "secretAccessKey": true, + "sessionToken": true, "endpoint": true, "serverHost": true, "serverSecure": true, "adapter": true, "concurrency": true, "env": true, } @@ -978,7 +1019,7 @@ var poolsExportCmd = &cobra.Command{ Short: "Export pool configuration", Long: `Export all pool configurations for the workspace as TOML. Writes to stdout by default. -Launcher secrets, such as a Kubernetes token, are redacted unless --include-secrets +Launcher secrets, such as a Kubernetes token or an AWS secret key, are redacted unless --include-secrets is given. A redacted file is refused by 'pools import', so it cannot silently clear the secrets it omits.`, RunE: runPoolsExport, @@ -987,7 +1028,7 @@ the secrets it omits.`, func init() { poolsExportCmd.Flags().StringVarP(&poolsExportOutput, "output", "o", "", "Output file (default: stdout)") poolsExportCmd.Flags().StringSliceVar(&poolsExportOnly, "only", nil, "Export only named pools") - poolsExportCmd.Flags().BoolVar(&poolsExportIncludeSecrets, "include-secrets", false, "Include launcher secrets (such as Kubernetes tokens) in the output") + poolsExportCmd.Flags().BoolVar(&poolsExportIncludeSecrets, "include-secrets", false, "Include launcher secrets (such as Kubernetes tokens or AWS secret keys) in the output") } func runPoolsExport(cmd *cobra.Command, args []string) error { @@ -1388,6 +1429,7 @@ func encodeInlineValue(buf *bytes.Buffer, v any) error { // camelCase to snake_case mapping for launcher fields var camelToSnake = map[string]string{ "dockerHost": "docker_host", + "networkMode": "network_mode", "serverHost": "server_host", "serverSecure": "server_secure", "serviceAccount": "service_account", @@ -1399,6 +1441,16 @@ var camelToSnake = map[string]string{ "caCert": "ca_cert", "activeDeadlineSeconds": "active_deadline_seconds", "volumeMounts": "volume_mounts", + "taskDefinition": "task_definition", + "containerName": "container_name", + "launchType": "launch_type", + "capacityProvider": "capacity_provider", + "securityGroups": "security_groups", + "assignPublicIp": "assign_public_ip", + "platformVersion": "platform_version", + "accessKeyId": "access_key_id", + "secretAccessKey": "secret_access_key", + "sessionToken": "session_token", } var snakeToCamel map[string]string diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index bf3cbf2f..8974cabf 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -137,7 +137,7 @@ Start a local server using Docker. | `--super-token-hash` | | | Pre-hashed super token (SHA-256 hex) | | `--secret` | | | Server secret for signing service tokens | | `--team` | | | Allowed team IDs for Studio auth | -| `--launcher` | | | Allowed launcher types (`docker`, `process`, `kubernetes`) | +| `--launcher` | | | Allowed launcher types (`docker`, `process`, `kubernetes`, `ecs`) | ## `coflux login` / `coflux logout` @@ -188,7 +188,7 @@ All manifest commands accept `--adapter` to specify the adapter command. | Flag | Description | |------|-------------| -| `--type` | Launcher type: `kubernetes`, `docker`, `process` (required) | +| `--type` | Launcher type: `kubernetes`, `docker`, `process`, `ecs` (required) | | `--set` | Set a field (e.g., `--set image=myapp:latest`, `--set env.KEY=VALUE`) | | `--modules`, `-m` | Modules to host | | `--provides` | Features workers provide | diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 984f0958..92767493 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -102,6 +102,67 @@ Note that the token is stored in the orchestration database. | `volumes` | Kubernetes volume definitions | | `volumeMounts` | Volume mounts in container | +#### ECS launcher + +:::note +The ECS launcher is experimental — the API may change based on feedback. +::: + +Launches workers as Amazon ECS tasks, on Fargate by default: + +```bash +coflux pools create mypool --type ecs \ + --set cluster=workers \ + --set taskDefinition=myapp-worker \ + --set region=eu-west-2 \ + --set subnets='["subnet-0123", "subnet-4567"]' \ + --set securityGroups=sg-0123 \ + --set serverHost=coflux.example.com \ + --modules myapp.workflows +``` + +The pool refers to an existing task definition, which is where the image, +CPU and memory, IAM roles and log configuration are set. For each worker, +one task is run from it, with the container's command overridden to the +modules to host and its environment to the worker's connection details. +So the container should run the Coflux worker with the modules left to +the command — an `ENTRYPOINT` of `coflux worker --adapter ...`, say. + +Workers connect out to the server, so a task needs a route to it and +nothing needs to reach the task: a public IP (`assignPublicIp`) in a +public subnet, or a NAT gateway from a private one. + +Credentials are taken from the pool when `accessKeyId` and +`secretAccessKey` are set (with `sessionToken` for temporary ones), and +otherwise from the server's surroundings the way the AWS SDKs look: +`AWS_ACCESS_KEY_ID` and friends in its environment, its ECS task role, +or its EC2 instance profile. They need `ecs:RunTask`, `ecs:DescribeTasks` +and `ecs:StopTask` on the cluster, `ecs:DescribeTaskDefinition` unless +`containerName` is set, and `iam:PassRole` for the roles the task +definition names. + +ECS doesn't expose container output through its API, so a worker's log +tail isn't shown; a task that fails to start reports its reason in its +place. Give the task definition a log configuration (`awslogs`, say) to +see what workers print. + +| Field | Description | +|-------|-------------| +| `cluster` | ECS cluster name or ARN | +| `taskDefinition` | Task definition family, `family:revision`, or ARN | +| `region` | AWS region | +| `containerName` | Container to override (default: the task definition's first) | +| `launchType` | `FARGATE` (default), `EC2`, or `EXTERNAL` | +| `capacityProvider` | Capacity provider to use instead of a launch type (e.g. `FARGATE_SPOT`) | +| `subnets` | Subnet IDs for the task (required on Fargate) | +| `securityGroups` | Security group IDs for the task | +| `assignPublicIp` | Give the task a public IP | +| `platformVersion` | Fargate platform version | +| `accessKeyId` | AWS access key ID | +| `secretAccessKey` | AWS secret access key | +| `sessionToken` | AWS session token, for temporary credentials | +| `endpoint` | ECS API endpoint override (e.g. a VPC endpoint) | + ### Common fields These fields apply to all launcher types: @@ -160,8 +221,8 @@ coflux pools export --only mypool --only gpu-pool -o pools.toml coflux pools import pools.toml ``` -Launcher secrets — currently the Kubernetes `token` — are redacted on export -unless `--include-secrets` is given. Importing a redacted file is refused rather +Launcher secrets — the Kubernetes `token`, and the ECS `secretAccessKey` and +`sessionToken` — are redacted on export unless `--include-secrets` is given. Importing a redacted file is refused rather than silently clearing the secrets it omits, so use `--include-secrets` when the exported file is meant to be imported again: diff --git a/docs/docs/server_config.md b/docs/docs/server_config.md index a8d8494b..3da68066 100644 --- a/docs/docs/server_config.md +++ b/docs/docs/server_config.md @@ -26,7 +26,7 @@ This is a convenience wrapper around `docker run`. Docker must be installed and | `--super-token` or `--super-token-hash` | _(none)_ | Set a super token (plain text or pre-hashed SHA-256 hex) | | `--secret` | _(none)_ | Server secret for signing service tokens | | `--team` | _(none)_ | Team IDs allowed for Studio auth (repeatable) | -| `--launcher` | _(none)_ | Allowed launcher types (repeatable, e.g. `docker`, `process`) | +| `--launcher` | _(none)_ | Allowed launcher types (repeatable: `docker`, `process`, `kubernetes`, `ecs`) | | `--image` | _(auto)_ | Docker image to use | ## Projects @@ -70,7 +70,7 @@ The server is configured via environment variables. When using `coflux server`, | `COFLUX_STUDIO_URL` | `https://studio.coflux.com` | Studio URL | | `COFLUX_DATA_DIR` | `./data` | Data directory path | | `COFLUX_ALLOW_ORIGINS` | `https://studio.coflux.com` | Comma-separated CORS origins | -| `COFLUX_LAUNCHER_TYPES` | _(none)_ | Allowed launcher types (e.g., `docker,process,kubernetes`) | +| `COFLUX_LAUNCHER_TYPES` | _(none)_ | Allowed launcher types (e.g., `docker,process,kubernetes,ecs`) | | `COFLUX_CLI_PATH` | `coflux` | CLI binary path for process launcher | ## Data storage diff --git a/server/lib/coflux/config.ex b/server/lib/coflux/config.ex index 22689efe..30e832d2 100644 --- a/server/lib/coflux/config.ex +++ b/server/lib/coflux/config.ex @@ -271,7 +271,7 @@ defmodule Coflux.Config do Defaults to an empty set (no launcher types enabled). Set via `COFLUX_LAUNCHER_TYPES` as a comma-separated list of type names - (e.g. `"docker,process"`). + (e.g. `"docker,process,kubernetes,ecs"`). """ def launcher_types do :persistent_term.get(:coflux_launcher_types) @@ -290,7 +290,7 @@ defmodule Coflux.Config do System.get_env("COFLUX_CLI_PATH", "coflux") end - @valid_launcher_types MapSet.new([:docker, :process, :kubernetes]) + @valid_launcher_types MapSet.new([:docker, :process, :kubernetes, :ecs]) defp parse_launcher_types do case System.get_env("COFLUX_LAUNCHER_TYPES") do diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index dd7ed5fb..14711876 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -11,6 +11,8 @@ defmodule Coflux.Handlers.Api do # again is refused rather than quietly clearing the real one. @redacted_secret "" + @ecs_launch_types ["FARGATE", "EC2", "EXTERNAL"] + # A directory upload arrives as one entry per file, so this bounds an # accidental drop of a very large tree. Unlike the sizes, which the # client asserts, the count is something the server can see for itself. @@ -1368,6 +1370,119 @@ defmodule Coflux.Handlers.Api do end end + defp parse_ecs_launcher(value) do + cluster = Map.get(value, "cluster") + task_definition = Map.get(value, "taskDefinition") + region = Map.get(value, "region") + container_name = Map.get(value, "containerName") + launch_type = Map.get(value, "launchType") + capacity_provider = Map.get(value, "capacityProvider") + subnets = wrap_list(Map.get(value, "subnets")) + security_groups = wrap_list(Map.get(value, "securityGroups")) + assign_public_ip = Map.get(value, "assignPublicIp") + platform_version = Map.get(value, "platformVersion") + access_key_id = Map.get(value, "accessKeyId") + secret_access_key = Map.get(value, "secretAccessKey") + session_token = Map.get(value, "sessionToken") + endpoint = Map.get(value, "endpoint") + + cond do + not is_binary(cluster) or cluster == "" or String.length(cluster) > 255 -> + {:error, :invalid} + + not is_binary(task_definition) or task_definition == "" or + String.length(task_definition) > 500 -> + {:error, :invalid} + + not is_binary(region) or not Regex.match?(~r/^[a-z0-9-]{1,30}$/, region) -> + {:error, :invalid} + + not is_nil(container_name) and + (not is_binary(container_name) or String.length(container_name) > 255) -> + {:error, :invalid} + + not is_nil(launch_type) and launch_type not in @ecs_launch_types -> + {:error, :invalid} + + not is_nil(capacity_provider) and + (not is_binary(capacity_provider) or String.length(capacity_provider) > 255) -> + {:error, :invalid} + + # A capacity provider strategy decides the launch type itself. + not is_nil(launch_type) and not is_nil(capacity_provider) -> + {:error, :invalid} + + not is_nil(subnets) and not is_string_list?(subnets, 16) -> + {:error, :invalid} + + not is_nil(security_groups) and not is_string_list?(security_groups, 5) -> + {:error, :invalid} + + not is_nil(assign_public_ip) and not is_boolean(assign_public_ip) -> + {:error, :invalid} + + not is_nil(platform_version) and + (not is_binary(platform_version) or String.length(platform_version) > 50) -> + {:error, :invalid} + + not is_nil(access_key_id) and + (not is_binary(access_key_id) or String.length(access_key_id) > 128) -> + {:error, :invalid} + + not is_nil(secret_access_key) and not is_binary(secret_access_key) -> + {:error, :invalid} + + secret_access_key == @redacted_secret -> + {:error, :redacted} + + not is_nil(session_token) and not is_binary(session_token) -> + {:error, :invalid} + + session_token == @redacted_secret -> + {:error, :redacted} + + # A key ID without its secret (or the reverse) can't sign anything, + # and a session token belongs to a key pair. + is_nil(access_key_id) != is_nil(secret_access_key) -> + {:error, :invalid} + + not is_nil(session_token) and is_nil(access_key_id) -> + {:error, :invalid} + + not is_nil(endpoint) and + (not is_binary(endpoint) or String.length(endpoint) > 500 or + not String.starts_with?(endpoint, ["http://", "https://"])) -> + {:error, :invalid} + + true -> + launcher = + %{type: :ecs, cluster: cluster, task_definition: task_definition, region: region} + |> maybe_put_value(:container_name, container_name) + |> maybe_put_value(:launch_type, launch_type) + |> maybe_put_value(:capacity_provider, capacity_provider) + |> maybe_put_value(:subnets, subnets) + |> maybe_put_value(:security_groups, security_groups) + |> maybe_put_value(:assign_public_ip, if(assign_public_ip == true, do: true)) + |> maybe_put_value(:platform_version, platform_version) + |> maybe_put_value(:access_key_id, access_key_id) + |> maybe_put_value(:secret_access_key, secret_access_key) + |> maybe_put_value(:session_token, session_token) + |> maybe_put_value(:endpoint, endpoint) + + {:ok, launcher} + end + end + + # A single ID is accepted where a list is expected, so `--set + # subnets=subnet-1` works without JSON. + defp wrap_list(value) when is_binary(value), do: [value] + defp wrap_list(value), do: value + + defp is_string_list?(value, max_length) do + is_list(value) and value != [] and length(value) <= max_length and + Enum.all?(value, &(is_binary(&1) and &1 != "")) + end + defp parse_common_launcher_fields(launcher, value) do server_host = Map.get(value, "serverHost") server_secure = Map.get(value, "serverSecure") @@ -1424,7 +1539,7 @@ defmodule Coflux.Handlers.Api do cond do is_map(value) -> case Map.fetch(value, "type") do - {:ok, type} when type in ["docker", "process", "kubernetes"] -> + {:ok, type} when type in ["docker", "process", "kubernetes", "ecs"] -> type_atom = String.to_existing_atom(type) if MapSet.member?(allowed, type_atom) do @@ -1433,6 +1548,7 @@ defmodule Coflux.Handlers.Api do "docker" -> parse_docker_launcher(value) "process" -> parse_process_launcher(value) "kubernetes" -> parse_kubernetes_launcher(value) + "ecs" -> parse_ecs_launcher(value) end) do parse_common_launcher_fields(launcher, value) end @@ -1516,6 +1632,31 @@ defmodule Coflux.Handlers.Api do :process -> %{"type" => "process", "directory" => launcher.directory} + :ecs -> + %{ + "type" => "ecs", + "cluster" => launcher.cluster, + "taskDefinition" => launcher.task_definition, + "region" => launcher.region + } + |> maybe_put_value("containerName", Map.get(launcher, :container_name)) + |> maybe_put_value("launchType", Map.get(launcher, :launch_type)) + |> maybe_put_value("capacityProvider", Map.get(launcher, :capacity_provider)) + |> maybe_put_value("subnets", Map.get(launcher, :subnets)) + |> maybe_put_value("securityGroups", Map.get(launcher, :security_groups)) + |> maybe_put_value("assignPublicIp", Map.get(launcher, :assign_public_ip)) + |> maybe_put_value("platformVersion", Map.get(launcher, :platform_version)) + |> maybe_put_value("accessKeyId", Map.get(launcher, :access_key_id)) + |> maybe_put_value( + "secretAccessKey", + secret_value(Map.get(launcher, :secret_access_key), include_secrets) + ) + |> maybe_put_value( + "sessionToken", + secret_value(Map.get(launcher, :session_token), include_secrets) + ) + |> maybe_put_value("endpoint", Map.get(launcher, :endpoint)) + :kubernetes -> %{"type" => "kubernetes", "image" => launcher.image} |> maybe_put_value("namespace", Map.get(launcher, :namespace)) @@ -1629,7 +1770,7 @@ defmodule Coflux.Handlers.Api do # If "type" is present, validate it; otherwise this is patching an existing launcher case Map.fetch(value, "type") do - {:ok, type} when type in ["docker", "process", "kubernetes"] -> + {:ok, type} when type in ["docker", "process", "kubernetes", "ecs"] -> type_atom = String.to_existing_atom(type) if MapSet.member?(allowed, type_atom) do @@ -1671,6 +1812,20 @@ defmodule Coflux.Handlers.Api do {"imagePullSecrets", &is_list/1}, {"hostAliases", &is_list/1}, {"resources", &is_map/1}, + {"cluster", &is_binary/1}, + {"taskDefinition", &is_binary/1}, + {"region", &is_binary/1}, + {"containerName", &is_binary/1}, + {"launchType", &(&1 in @ecs_launch_types)}, + {"capacityProvider", &is_binary/1}, + {"subnets", &(is_binary(&1) or is_string_list?(&1, 16))}, + {"securityGroups", &(is_binary(&1) or is_string_list?(&1, 5))}, + {"assignPublicIp", &is_boolean/1}, + {"platformVersion", &is_binary/1}, + {"accessKeyId", &is_binary/1}, + {"secretAccessKey", &is_binary/1}, + {"sessionToken", &is_binary/1}, + {"endpoint", &is_binary/1}, {"serverHost", &is_binary/1}, {"serverSecure", &is_boolean/1}, {"adapter", fn v -> is_list(v) and v != [] and Enum.all?(v, &is_binary/1) end}, @@ -1703,6 +1858,20 @@ defmodule Coflux.Handlers.Api do "imagePullSecrets" => :image_pull_secrets, "hostAliases" => :host_aliases, "resources" => :resources, + "cluster" => :cluster, + "taskDefinition" => :task_definition, + "region" => :region, + "containerName" => :container_name, + "launchType" => :launch_type, + "capacityProvider" => :capacity_provider, + "subnets" => :subnets, + "securityGroups" => :security_groups, + "assignPublicIp" => :assign_public_ip, + "platformVersion" => :platform_version, + "accessKeyId" => :access_key_id, + "secretAccessKey" => :secret_access_key, + "sessionToken" => :session_token, + "endpoint" => :endpoint, "serverHost" => :server_host, "serverSecure" => :server_secure, "adapter" => :adapter, @@ -1722,13 +1891,18 @@ defmodule Coflux.Handlers.Api do {:ok, field_value} -> if validator.(field_value) do processed_value = - if json_key == "env" and is_map(field_value) do - Map.new(field_value, fn - {k, nil} -> {k, :unset} - {k, v} -> {k, v} - end) - else - field_value + cond do + json_key == "env" and is_map(field_value) -> + Map.new(field_value, fn + {k, nil} -> {k, :unset} + {k, v} -> {k, v} + end) + + json_key in ["subnets", "securityGroups"] -> + wrap_list(field_value) + + true -> + field_value end {:cont, {:ok, Map.put(acc, atom_key, processed_value)}} diff --git a/server/lib/coflux/launchers/aws_credentials.ex b/server/lib/coflux/launchers/aws_credentials.ex new file mode 100644 index 00000000..d85ed0df --- /dev/null +++ b/server/lib/coflux/launchers/aws_credentials.ex @@ -0,0 +1,171 @@ +defmodule Coflux.Launchers.AwsCredentials do + @moduledoc """ + Finds AWS credentials for a launcher to sign requests with. + + Credentials configured on the pool win. Otherwise the server's own + surroundings are searched the way the AWS SDKs do, in the same order: + environment variables, then the container credentials endpoint (an ECS + task role, when the server itself runs on ECS), then the EC2 instance + metadata service (an instance profile). Nothing is cached: the + endpoints are link-local and quick, and a launcher call is rare enough + that a lookup per call costs less than getting expiry right. + """ + + @container_credentials_host "http://169.254.170.2" + @instance_metadata_endpoint "http://169.254.169.254" + @instance_metadata_token_ttl "21600" + + # These endpoints are on the local link, or not there at all: a slow + # answer means the latter, and a launcher task shouldn't sit on it. + @connect_timeout_ms 1_000 + @receive_timeout_ms 2_000 + + @type t :: %{ + access_key_id: String.t(), + secret_access_key: String.t(), + session_token: String.t() | nil + } + + @doc """ + Resolves credentials, preferring `static` (a map with `:access_key_id`, + `:secret_access_key` and optionally `:session_token`) when given. + """ + @spec resolve(map() | nil) :: {:ok, t()} | {:error, :credentials_missing} + def resolve(%{access_key_id: access_key_id, secret_access_key: secret_access_key} = static) + when is_binary(access_key_id) and is_binary(secret_access_key) do + {:ok, + %{ + access_key_id: access_key_id, + secret_access_key: secret_access_key, + session_token: Map.get(static, :session_token) + }} + end + + def resolve(_static) do + Enum.find_value( + [&from_environment/0, &from_container/0, &from_instance_metadata/0], + {:error, :credentials_missing}, + fn source -> + case source.() do + {:ok, credentials} -> {:ok, credentials} + :none -> nil + end + end + ) + end + + defp from_environment do + access_key_id = System.get_env("AWS_ACCESS_KEY_ID") + secret_access_key = System.get_env("AWS_SECRET_ACCESS_KEY") + + if present?(access_key_id) and present?(secret_access_key) do + {:ok, + %{ + access_key_id: access_key_id, + secret_access_key: secret_access_key, + session_token: blank_to_nil(System.get_env("AWS_SESSION_TOKEN")) + }} + else + :none + end + end + + defp from_container do + url = + cond do + present?(System.get_env("AWS_CONTAINER_CREDENTIALS_FULL_URI")) -> + System.get_env("AWS_CONTAINER_CREDENTIALS_FULL_URI") + + present?(System.get_env("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI")) -> + @container_credentials_host <> System.get_env("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI") + + true -> + nil + end + + if url do + case request(:get, url, container_auth_headers()) do + {:ok, body} -> parse_credentials(body) + :error -> :none + end + else + :none + end + end + + defp container_auth_headers do + token = + cond do + present?(System.get_env("AWS_CONTAINER_AUTHORIZATION_TOKEN")) -> + System.get_env("AWS_CONTAINER_AUTHORIZATION_TOKEN") + + present?(System.get_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")) -> + case File.read(System.get_env("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE")) do + {:ok, contents} -> String.trim(contents) + {:error, _} -> nil + end + + true -> + nil + end + + if token, do: [{"authorization", token}], else: [] + end + + defp from_instance_metadata do + if System.get_env("AWS_EC2_METADATA_DISABLED") in ["true", "1"] do + :none + else + base = System.get_env("AWS_EC2_METADATA_SERVICE_ENDPOINT") || @instance_metadata_endpoint + credentials_url = base <> "/latest/meta-data/iam/security-credentials/" + + with {:ok, token} <- + request(:put, base <> "/latest/api/token", [ + {"x-aws-ec2-metadata-token-ttl-seconds", @instance_metadata_token_ttl} + ]), + token_headers = [{"x-aws-ec2-metadata-token", token}], + {:ok, roles} <- request(:get, credentials_url, token_headers), + [role | _] <- roles |> String.split("\n", trim: true) |> Enum.map(&String.trim/1), + {:ok, body} <- request(:get, credentials_url <> role, token_headers) do + parse_credentials(body) + else + _ -> :none + end + end + end + + defp parse_credentials(body) do + case Jason.decode(body) do + {:ok, %{"AccessKeyId" => access_key_id, "SecretAccessKey" => secret_access_key} = decoded} + when is_binary(access_key_id) and is_binary(secret_access_key) -> + {:ok, + %{ + access_key_id: access_key_id, + secret_access_key: secret_access_key, + session_token: blank_to_nil(decoded["Token"]) + }} + + _ -> + :none + end + end + + defp request(method, url, headers) do + case Req.request( + method: method, + url: url, + headers: headers, + retry: false, + decode_body: false, + connect_options: [timeout: @connect_timeout_ms], + receive_timeout: @receive_timeout_ms + ) do + {:ok, %{status: 200, body: body}} when is_binary(body) -> {:ok, body} + _ -> :error + end + end + + defp present?(value), do: is_binary(value) and value != "" + + defp blank_to_nil(value), do: if(present?(value), do: value, else: nil) +end diff --git a/server/lib/coflux/launchers/ecs.ex b/server/lib/coflux/launchers/ecs.ex new file mode 100644 index 00000000..df709bd2 --- /dev/null +++ b/server/lib/coflux/launchers/ecs.ex @@ -0,0 +1,470 @@ +defmodule Coflux.EcsLauncher do + @moduledoc """ + Runs workers as Amazon ECS tasks. + + A pool names a task definition, and each worker is one task run from it, + with the container's command overridden to the modules to host and its + environment to what the worker needs to connect. Everything else about + the task - image, CPU and memory, IAM roles, logging - belongs to the + task definition, which is where ECS users expect to configure it, so the + launcher doesn't try to own it. + + ECS has no log API of its own: container output goes wherever the task + definition's log configuration sends it. What a stopped task does carry + is a `stoppedReason`, which is the diagnostic when a task never started + (an image that couldn't be pulled, say), so that stands in for the log + tail. + + Credentials are resolved on every call rather than kept with the task: + the ones the server finds in its surroundings rotate, and the static + ones a pool can be configured with are the only kind that are safe to + remember. See `Coflux.Launchers.AwsCredentials`. + """ + + import Coflux.Launchers.Utils, only: [truncate_bytes: 2] + + alias Coflux.Launchers.AwsCredentials + + @api_target_prefix "AmazonEC2ContainerServiceV20141113" + @reason_max_bytes 1024 + + # ECS limits `startedBy` to 36 characters. + @started_by_max_length 36 + + @unauthorized_errors [ + "UnrecognizedClientException", + "InvalidSignatureException", + "SignatureDoesNotMatch", + "ExpiredTokenException", + "ExpiredToken", + "InvalidClientTokenId", + "InvalidAccessKeyId", + "IncompleteSignature", + "MissingAuthenticationToken" + ] + + @invalid_errors [ + "InvalidParameterException", + "PlatformTaskDefinitionIncompatibilityException", + "PlatformUnknownException", + "UnsupportedFeatureException", + "BlockedException" + ] + + def launch(env, modules, config, opts \\ %{}) do + with {:ok, conn} <- build_conn(config), + {:ok, container_name} <- resolve_container_name(conn, config), + {:ok, task_arn} <- run_task(conn, config, container_name, env, modules, opts) do + data = %{ + task_arn: task_arn, + cluster: Map.fetch!(config, :cluster), + container_name: container_name, + region: conn.region + } + + data = + data + |> maybe_put(:endpoint, config[:endpoint]) + |> maybe_put(:credentials, static_credentials(config)) + + {:ok, data} + else + {:error, reason} -> + case normalize_launch_error(reason) do + {error, nil} -> {:error, error} + {error, detail} -> {:error, error, detail} + end + end + end + + def stop(%{task_arn: task_arn, cluster: cluster} = data) do + with {:ok, conn} <- conn_from_data(data), + {:ok, _body} <- + ecs_request(conn, "StopTask", %{ + "cluster" => cluster, + "task" => task_arn, + "reason" => "Stopped by Coflux" + }) do + :ok + else + # Already gone, which is what was being asked for. + {:error, {:api, _status, "InvalidParameterException", message}} + when is_binary(message) -> + if message =~ ~r/not found/i, do: :ok, else: {:error, "stop_invalid"} + + {:error, reason} -> + {:error, describe_error(reason)} + end + end + + def poll(%{task_arn: task_arn, cluster: cluster} = data) do + with {:ok, conn} <- conn_from_data(data), + {:ok, body} <- + ecs_request(conn, "DescribeTasks", %{"cluster" => cluster, "tasks" => [task_arn]}) do + case body do + %{"tasks" => [task | _]} -> + interpret_task(task, data) + + # Stopped tasks are only described for an hour or so afterwards; + # one that has aged out has nothing left to say. + %{"failures" => [%{"reason" => "MISSING"} | _]} -> + {:ok, false, nil, nil} + + %{"failures" => [%{"reason" => reason} | _]} -> + {:error, "describe_failed:#{reason}"} + + _ -> + {:error, "unexpected_response"} + end + else + {:error, reason} -> {:error, describe_error(reason)} + end + end + + # --- Task state --- + + defp interpret_task(%{"lastStatus" => "STOPPED"} = task, data) do + error = stop_error(task, data[:container_name]) + logs = if error, do: stopped_reason(task) + {:ok, false, error, logs} + end + + # Anything else - provisioning, pending, running, or on its way to + # stopped - is a task that hasn't finished yet. + defp interpret_task(_task, _data), do: {:ok, true} + + # Returns nil for a task that stopped because it was asked to, or an + # error code for one that didn't. + defp stop_error(task, container_name) do + case task["stopCode"] do + "UserInitiated" -> nil + "ServiceSchedulerInitiated" -> nil + "SpotInterruption" -> "spot_interrupted" + "TerminationNotice" -> "spot_interrupted" + "TaskFailedToStart" -> failed_to_start_error(task["stoppedReason"]) + "EssentialContainerExited" -> container_exit_error(task["containers"], container_name) + _ -> container_exit_error(task["containers"], container_name) || generic_stop_error(task) + end + end + + defp failed_to_start_error(reason) when is_binary(reason) do + cond do + reason =~ "CannotPull" -> "image_pull_error" + reason =~ "ResourceInitializationError" -> "resource_initialization_error" + reason =~ "CannotCreateContainerError" -> "container_start_error" + reason =~ "CannotStartContainerError" -> "container_start_error" + true -> "task_failed_to_start" + end + end + + defp failed_to_start_error(_reason), do: "task_failed_to_start" + + defp container_exit_error(containers, container_name) when is_list(containers) do + container = + Enum.find(containers, &(&1["name"] == container_name)) || + Enum.find(containers, &is_integer(&1["exitCode"])) + + case container do + %{"reason" => reason} when is_binary(reason) -> + if reason =~ "OutOfMemory", do: "oom_killed", else: exit_code_error(container) + + %{} -> + exit_code_error(container) + + nil -> + "container_exited" + end + end + + defp container_exit_error(_containers, _container_name), do: "container_exited" + + defp exit_code_error(%{"exitCode" => 0}), do: nil + defp exit_code_error(%{"exitCode" => code}) when is_integer(code), do: "exit_code:#{code}" + defp exit_code_error(_container), do: "container_exited" + + defp generic_stop_error(%{"stoppedReason" => reason}) when is_binary(reason), do: "task_stopped" + defp generic_stop_error(_task), do: nil + + defp stopped_reason(%{"stoppedReason" => reason}) when is_binary(reason) and reason != "", + do: truncate_bytes(reason, @reason_max_bytes) + + defp stopped_reason(_task), do: nil + + # --- Launching --- + + defp resolve_container_name(_conn, %{container_name: name}) when is_binary(name), + do: {:ok, name} + + # Without a name to override, the task definition says what its + # containers are called; a worker task has one. + defp resolve_container_name(conn, config) do + case ecs_request(conn, "DescribeTaskDefinition", %{ + "taskDefinition" => Map.fetch!(config, :task_definition) + }) do + {:ok, %{"taskDefinition" => %{"containerDefinitions" => [%{"name" => name} | _]}}} + when is_binary(name) -> + {:ok, name} + + {:ok, _body} -> + {:error, :no_container} + + # A task definition that doesn't exist isn't reported as not found + # here, only as not describable. + {:error, {:api, _status, "ClientException", message}} -> + {:error, {:task_definition_not_found, message}} + + {:error, reason} -> + {:error, reason} + end + end + + defp run_task(conn, config, container_name, env, modules, opts) do + override = %{ + "name" => container_name, + "environment" => Enum.map(env, fn {name, value} -> %{"name" => name, "value" => value} end) + } + + override = if modules == [], do: override, else: Map.put(override, "command", modules) + + body = + %{ + "cluster" => Map.fetch!(config, :cluster), + "taskDefinition" => Map.fetch!(config, :task_definition), + "count" => 1, + "overrides" => %{"containerOverrides" => [override]}, + "startedBy" => started_by(opts) + } + |> put_launch_type(config) + |> put_network_configuration(config) + |> maybe_put("platformVersion", config[:platform_version]) + + case ecs_request(conn, "RunTask", body) do + {:ok, %{"tasks" => [%{"taskArn" => task_arn} | _]}} when is_binary(task_arn) -> + {:ok, task_arn} + + # No task and no HTTP error: the request was fine, but nothing could + # place it (no capacity, typically). + {:ok, %{"failures" => [%{"reason" => reason} = failure | _]}} -> + {:error, {:run_task_failure, reason, failure["detail"]}} + + {:ok, _body} -> + {:error, :unexpected_response} + + {:error, reason} -> + {:error, reason} + end + end + + defp put_launch_type(body, config) do + case config[:capacity_provider] do + nil -> + Map.put(body, "launchType", config[:launch_type] || "FARGATE") + + provider -> + Map.put(body, "capacityProviderStrategy", [ + %{"capacityProvider" => provider, "weight" => 1} + ]) + end + end + + defp put_network_configuration(body, config) do + case config[:subnets] do + subnets when is_list(subnets) and subnets != [] -> + awsvpc = + %{ + "subnets" => subnets, + "assignPublicIp" => if(config[:assign_public_ip], do: "ENABLED", else: "DISABLED") + } + |> maybe_put("securityGroups", config[:security_groups]) + + Map.put(body, "networkConfiguration", %{"awsvpcConfiguration" => awsvpc}) + + _ -> + body + end + end + + defp started_by(opts) do + "coflux:#{Map.get(opts, :pool_name, "worker")}" + |> String.slice(0, @started_by_max_length) + end + + # --- Errors --- + + # Returns {code, detail}: the code is what the worker is deactivated + # with, the detail (when there is one) is the API's own message, which + # for an invalid request is the only thing that says what was wrong. + defp normalize_launch_error(:credentials_missing), do: {"launch_credentials_missing", nil} + defp normalize_launch_error(:request_failed), do: {"launch_request_failed", nil} + defp normalize_launch_error(:no_container), do: {"launch_no_container", nil} + defp normalize_launch_error(:unexpected_response), do: {"launch_api_error", nil} + + defp normalize_launch_error({:run_task_failure, reason, detail}), + do: {"launch_failed:#{reason}", detail} + + defp normalize_launch_error({:task_definition_not_found, message}), + do: {"launch_task_definition_not_found", message} + + defp normalize_launch_error({:api, _status, type, message}) do + code = + cond do + type == "ClusterNotFoundException" -> + "launch_cluster_not_found" + + type == "ClientException" and task_definition_not_found?(message) -> + "launch_task_definition_not_found" + + type == "ClientException" -> + "launch_invalid" + + type in @invalid_errors -> + "launch_invalid" + + type == "AccessDeniedException" -> + "launch_forbidden" + + type in @unauthorized_errors -> + "launch_unauthorized" + + type in ["ThrottlingException", "TooManyRequestsException"] -> + "launch_throttled" + + type == "ServerException" -> + "launch_server_error" + + true -> + "launch_api_error" + end + + {code, message} + end + + defp normalize_launch_error(_reason), do: {"launch_api_error", nil} + + defp task_definition_not_found?(message) when is_binary(message), + do: message =~ ~r/task ?definition/i and message =~ ~r/not found/i + + defp task_definition_not_found?(_message), do: false + + # For poll and stop, where the error is retried rather than shown. + defp describe_error(:credentials_missing), do: "credentials_missing" + defp describe_error(:request_failed), do: "request_failed" + + defp describe_error({:api, _status, type, _message}) when is_binary(type), + do: "api_error:#{type}" + + defp describe_error({:api, status, _type, _message}), do: "api_status:#{status}" + defp describe_error(reason) when is_binary(reason), do: reason + defp describe_error(reason), do: inspect(reason) + + # --- Connection --- + + defp build_conn(config) do + region = Map.fetch!(config, :region) + + with {:ok, credentials} <- AwsCredentials.resolve(static_credentials(config)) do + {:ok, + %{ + region: region, + endpoint: config[:endpoint] || default_endpoint(region), + credentials: credentials + }} + end + end + + defp conn_from_data(data) do + region = Map.fetch!(data, :region) + + with {:ok, credentials} <- AwsCredentials.resolve(data[:credentials]) do + {:ok, + %{ + region: region, + endpoint: data[:endpoint] || default_endpoint(region), + credentials: credentials + }} + end + end + + defp default_endpoint(region), do: "https://ecs.#{region}.amazonaws.com" + + defp static_credentials( + %{access_key_id: access_key_id, secret_access_key: secret_access_key} = config + ) + when is_binary(access_key_id) and is_binary(secret_access_key) do + %{access_key_id: access_key_id, secret_access_key: secret_access_key} + |> maybe_put(:session_token, config[:session_token]) + end + + defp static_credentials(_config), do: nil + + # --- ECS API --- + + # The ECS API is JSON 1.1 over HTTPS: every call is a POST to the + # regional endpoint, and the header says which operation. + defp ecs_request(conn, action, body) do + credentials = conn.credentials + + sigv4 = + [ + service: "ecs", + region: conn.region, + access_key_id: credentials.access_key_id, + secret_access_key: credentials.secret_access_key + ] + |> maybe_put_keyword(:token, credentials[:session_token]) + + request = + [ + method: :post, + url: conn.endpoint, + headers: [ + {"content-type", "application/x-amz-json-1.1"}, + {"x-amz-target", "#{@api_target_prefix}.#{action}"} + ], + body: Jason.encode!(body), + aws_sigv4: sigv4, + retry: false, + decode_body: false + ] + + case Req.request(request) do + {:ok, %{status: status, body: raw}} -> + decoded = + case Jason.decode(raw) do + {:ok, map} when is_map(map) -> map + _ -> %{} + end + + if status in 200..299 do + {:ok, decoded} + else + {:error, {:api, status, error_type(decoded), decoded["message"] || decoded["Message"]}} + end + + {:error, _exception} -> + {:error, :request_failed} + end + end + + # The type comes back as e.g. "ClusterNotFoundException", or namespaced + # as "com.amazonaws.ecs#ClusterNotFoundException", or with a suffix as + # "ClusterNotFoundException:http://...". + defp error_type(%{"__type" => type}) when is_binary(type) do + type + |> String.split("#") + |> List.last() + |> String.split(":") + |> hd() + end + + defp error_type(_body), do: nil + + # --- Helpers --- + + defp maybe_put(map, _key, nil), do: map + defp maybe_put(map, _key, []), do: map + defp maybe_put(map, key, value), do: Map.put(map, key, value) + + defp maybe_put_keyword(keyword, _key, nil), do: keyword + defp maybe_put_keyword(keyword, key, value), do: Keyword.put(keyword, key, value) +end diff --git a/server/lib/coflux/orchestration/server/fleet.ex b/server/lib/coflux/orchestration/server/fleet.ex index a5a2327d..793662d3 100644 --- a/server/lib/coflux/orchestration/server/fleet.ex +++ b/server/lib/coflux/orchestration/server/fleet.ex @@ -390,6 +390,7 @@ defmodule Coflux.Orchestration.Server.Fleet do :docker -> Coflux.DockerLauncher :process -> Coflux.ProcessLauncher :kubernetes -> Coflux.KubernetesLauncher + :ecs -> Coflux.EcsLauncher end task = Task.Supervisor.async_nolink(Coflux.LauncherSupervisor, module, fun, args) diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 50820810..345c2cb4 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -523,11 +523,15 @@ defmodule Coflux.Orchestration.Server.Scheduler do %{pool_name: pool_name} ], fn state, result -> - {data, error} = + # A launcher can say more than a code about why a + # launch failed - an API's own message, typically - + # and that goes where a log tail would. + {data, error, detail} = case result do - {:ok, {:ok, data}} -> {data, nil} - {:ok, {:error, error}} -> {nil, error} - :error -> {nil, "launch_crashed"} + {:ok, {:ok, data}} -> {data, nil, nil} + {:ok, {:error, error}} -> {nil, error, nil} + {:ok, {:error, error, detail}} -> {nil, error, detail} + :error -> {nil, "launch_crashed", nil} end {:ok, started_at} = @@ -550,7 +554,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do # relaunches on the very next pass. state |> record_pool_launch_failure(pool_id) - |> Fleet.deactivate_worker(worker_id, error) + |> Fleet.deactivate_worker(worker_id, error, detail) Map.has_key?(state.workers, worker_id) -> put_in( diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index 794bc558..c2429ade 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -998,6 +998,7 @@ defmodule Coflux.Orchestration.Workspaces do :process -> 0 :docker -> 1 :kubernetes -> 2 + :ecs -> 3 end end @@ -1006,6 +1007,7 @@ defmodule Coflux.Orchestration.Workspaces do 0 -> :process 1 -> :docker 2 -> :kubernetes + 3 -> :ecs end end diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index 82021763..d8afa1bc 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -65,6 +65,26 @@ defmodule Coflux.Topics.Pool do # server's host rather than a credential, so it stays. Everything # else a pool is configured with belongs here too, or `pools get` # shows less than `pools export` does. + # As above: the secret access key and session token are credentials, + # so they stay out. The key ID says which credentials without being + # one, and is how a pool's access is recognised, so it stays. + :ecs -> + %{ + type: "ecs", + cluster: launcher.cluster, + taskDefinition: launcher.task_definition, + region: launcher.region + } + |> maybe_put(:containerName, Map.get(launcher, :container_name)) + |> maybe_put(:launchType, Map.get(launcher, :launch_type)) + |> maybe_put(:capacityProvider, Map.get(launcher, :capacity_provider)) + |> maybe_put(:subnets, Map.get(launcher, :subnets)) + |> maybe_put(:securityGroups, Map.get(launcher, :security_groups)) + |> maybe_put(:assignPublicIp, Map.get(launcher, :assign_public_ip)) + |> maybe_put(:platformVersion, Map.get(launcher, :platform_version)) + |> maybe_put(:accessKeyId, Map.get(launcher, :access_key_id)) + |> maybe_put(:endpoint, Map.get(launcher, :endpoint)) + :kubernetes -> %{type: "kubernetes", image: launcher.image} |> maybe_put(:namespace, Map.get(launcher, :namespace)) diff --git a/tests/support/ecs.py b/tests/support/ecs.py new file mode 100644 index 00000000..00f93f05 --- /dev/null +++ b/tests/support/ecs.py @@ -0,0 +1,236 @@ +"""A stand-in for the Amazon ECS API, for exercising the ECS launcher. + +The real service runs containers. This runs ``coflux worker`` processes +from the command and environment overrides that a RunTask carries, and +describes them the way ECS describes tasks, so the launcher's whole path - +request signing, the RunTask it builds, polling, stopping, and how a +stopped task's state is read - is exercised without an AWS account. + +Requests aren't authenticated (there's no secret to check against), but +what the launcher sends is kept for tests to inspect. +""" + +import json +import os +import signal +import subprocess +import threading +import time +import uuid +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +TARGET_PREFIX = "AmazonEC2ContainerServiceV20141113." + + +class FakeEcs: + def __init__(self, cli_path, cwd, cluster="test-cluster", container_name="worker"): + self.cli_path = cli_path + self.cwd = cwd + self.cluster = cluster + self.container_name = container_name + # (action, body, headers) in the order received. + self.requests: list[tuple[str, dict, dict]] = [] + # task ARN -> {"proc", "stop_code", "stopped_reason", "container_reason", + # "exit_code"} + self.tasks = {} + # (type, message) to reject every RunTask with, or None. + self.run_task_error = None + # Likewise for DescribeTaskDefinition. + self.task_definition_error = None + self._lock = threading.Lock() + self._server = ThreadingHTTPServer(("127.0.0.1", 0), self._make_handler()) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + @property + def endpoint(self): + return f"http://127.0.0.1:{self._server.server_port}" + + def start(self): + self._thread.start() + + def close(self): + self._server.shutdown() + self._server.server_close() + with self._lock: + tasks = list(self.tasks.values()) + for task in tasks: + proc = task["proc"] + if proc.poll() is None: + proc.kill() + proc.wait(timeout=5) + + def requests_for(self, action): + with self._lock: + return [ + (body, headers) for a, body, headers in self.requests if a == action + ] + + def wait_for(self, action, count=1, timeout=30): + """Wait until ``count`` requests for ``action`` have been received.""" + deadline = time.time() + timeout + while time.time() < deadline: + received = self.requests_for(action) + if len(received) >= count: + return received + time.sleep(0.1) + raise TimeoutError(f"{action} not received {count} time(s) within {timeout}s") + + def task_arns(self): + with self._lock: + return list(self.tasks) + + def kill_with_oom(self, arn): + """Kill a task's process the way an OOM kill looks from ECS.""" + with self._lock: + task = self.tasks[arn] + task["stop_code"] = "EssentialContainerExited" + task["stopped_reason"] = "Essential container in task exited" + task["container_reason"] = ( + "OutOfMemoryError: Container killed due to memory usage" + ) + task["exit_code"] = 137 + proc = task["proc"] + proc.send_signal(signal.SIGKILL) + proc.wait(timeout=10) + + # --- Handling --- + + def _make_handler(self): + fake = self + + class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length) or b"{}") + target = self.headers.get("X-Amz-Target", "") + action = ( + target[len(TARGET_PREFIX) :] + if target.startswith(TARGET_PREFIX) + else target + ) + headers = {k.lower(): v for k, v in self.headers.items()} + with fake._lock: + fake.requests.append((action, body, headers)) + status, response = fake._dispatch(action, body) + payload = json.dumps(response).encode() + self.send_response(status) + self.send_header("Content-Type", "application/x-amz-json-1.1") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + return Handler + + def _dispatch(self, action, body): + if action == "DescribeTaskDefinition": + if self.task_definition_error: + error_type, message = self.task_definition_error + return 400, {"__type": error_type, "message": message} + return 200, { + "taskDefinition": { + "family": body.get("taskDefinition"), + "containerDefinitions": [{"name": self.container_name}], + } + } + if action == "RunTask": + return self._run_task(body) + if action == "DescribeTasks": + return self._describe_tasks(body) + if action == "StopTask": + return self._stop_task(body) + return 400, { + "__type": "InvalidParameterException", + "message": f"Unknown action {action}", + } + + def _run_task(self, body): + if self.run_task_error: + error_type, message = self.run_task_error + return 400, {"__type": error_type, "message": message} + if body.get("cluster") != self.cluster: + return 400, { + "__type": "ClusterNotFoundException", + "message": "Cluster not found.", + } + + override = body["overrides"]["containerOverrides"][0] + command = override.get("command", []) + env = {e["name"]: e["value"] for e in override.get("environment", [])} + + task_id = uuid.uuid4().hex + # A container has only the environment its task gives it - not the + # test runner's shell - and ECS sends its output to the task + # definition's log driver; here that's a file next to the worker, + # for when a test needs to see why one didn't start. + with open(os.path.join(self.cwd, f"task-{task_id}.log"), "wb") as log: + proc = subprocess.Popen( + [self.cli_path, "worker", *command], + cwd=self.cwd, + env={"PATH": os.environ["PATH"], **env}, + stdout=log, + stderr=subprocess.STDOUT, + ) + arn = f"arn:aws:ecs:us-east-1:123456789012:task/{self.cluster}/{task_id}" + with self._lock: + self.tasks[arn] = { + "proc": proc, + "stop_code": None, + "stopped_reason": None, + "container_reason": None, + "exit_code": None, + } + return 200, { + "tasks": [{"taskArn": arn, "lastStatus": "PROVISIONING"}], + "failures": [], + } + + def _describe_tasks(self, body): + tasks = [] + failures = [] + for arn in body.get("tasks", []): + with self._lock: + task = self.tasks.get(arn) + if task is None: + failures.append({"arn": arn, "reason": "MISSING"}) + else: + tasks.append(self._describe(arn, task)) + return 200, {"tasks": tasks, "failures": failures} + + def _stop_task(self, body): + arn = body.get("task") + with self._lock: + task = self.tasks.get(arn) + if task is not None and task["stop_code"] is None: + task["stop_code"] = "UserInitiated" + task["stopped_reason"] = body.get("reason", "") + if task is None: + return 400, { + "__type": "InvalidParameterException", + "message": "The referenced task was not found.", + } + proc = task["proc"] + if proc.poll() is None: + proc.terminate() + return 200, {"task": self._describe(arn, task)} + + def _describe(self, arn, task): + proc = task["proc"] + returncode = proc.poll() + if returncode is None: + return {"taskArn": arn, "lastStatus": "RUNNING", "containers": []} + exit_code = task["exit_code"] if task["exit_code"] is not None else returncode + container = {"name": self.container_name, "exitCode": exit_code} + if task["container_reason"]: + container["reason"] = task["container_reason"] + described = { + "taskArn": arn, + "lastStatus": "STOPPED", + "stopCode": task["stop_code"] or "EssentialContainerExited", + "containers": [container], + } + if task["stopped_reason"]: + described["stoppedReason"] = task["stopped_reason"] + return described diff --git a/tests/support/server.py b/tests/support/server.py index 167b6e7a..7dd8ac67 100644 --- a/tests/support/server.py +++ b/tests/support/server.py @@ -83,8 +83,9 @@ def _start_local(self, timeout): "COFLUX_PUBLIC_HOST": "%.localhost:" + str(self.port), "COFLUX_REQUIRE_AUTH": "false", # kubernetes is allowed so that its configuration can be - # exercised; storing a pool config contacts no cluster. - "COFLUX_LAUNCHER_TYPES": "process,docker,kubernetes", + # exercised; storing a pool config contacts no cluster. ecs + # is exercised against a stand-in for the API (support.ecs). + "COFLUX_LAUNCHER_TYPES": "process,docker,kubernetes,ecs", "COFLUX_SUPER_TOKEN_HASH": hashlib.sha256(SUPER_TOKEN.encode()).hexdigest(), "COFLUX_CLI_PATH": cli_path, **self._extra_env, diff --git a/tests/test_pools.py b/tests/test_pools.py index 5729fc7e..983a4b59 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -5,10 +5,13 @@ """ import json +import os import subprocess +import time import pytest from support import cli +from support.ecs import FakeEcs from support.executor import Executor from support.helpers import ADAPTER_SCRIPT, poll_result from support.manifest import manifest, task, workflow @@ -597,3 +600,301 @@ def test_importing_a_redacted_export_is_refused(self, pool_env, tmp_path): # And the real token is untouched. assert "super-secret-token" in cli.pools_export(include_secrets=True, host=host) + + +# --------------------------------------------------------------------------- +# ECS launcher + + +@pytest.fixture +def ecs_env(pool_env): + """A pool environment with a stand-in for the ECS API (see support.ecs).""" + cli_path = os.path.abspath(os.environ.get("COFLUX_BIN", "coflux")) + fake = FakeEcs(cli_path, cwd=str(pool_env["worker_dir"])) + fake.start() + try: + yield {**pool_env, "ecs": fake} + finally: + fake.close() + + +def _setup_ecs_pool(ecs_env, targets, modules=None, pool_name="ecs-pool", sets=()): + """Write the manifest and create an ECS pool pointed at the fake API. + + ``sets`` are extra ``--set`` fields; a later one overrides a default. + """ + modules = modules or ["test"] + host = ecs_env["host"] + fake = ecs_env["ecs"] + + with open(ecs_env["manifest_path"], "w") as f: + json.dump(manifest(targets), f) + + adapter = [ + "python3", + ADAPTER_SCRIPT, + "--manifest", + ecs_env["manifest_path"], + "--socket", + ecs_env["socket_path"], + ] + + fields = [ + f"cluster={fake.cluster}", + "taskDefinition=worker-task", + "region=us-east-1", + f"endpoint={fake.endpoint}", + "accessKeyId=AKIATEST", + "secretAccessKey=test-secret-key", + 'subnets=["subnet-1", "subnet-2"]', + "securityGroups=sg-1", + "assignPublicIp=true", + f"adapter={json.dumps(adapter)}", + *sets, + ] + args = ["pools", "create", pool_name, "--type", "ecs"] + for field in fields: + args.extend(["--set", field]) + args.extend(["--modules", ",".join(modules)]) + cli._coflux(*args, host=host, output=None) + + cli.manifests_register(*modules, adapter=",".join(adapter), host=host) + + +def _wait_for_worker(host, pool_name, predicate, timeout=30): + """Poll the pool's launches until a worker satisfies the predicate.""" + deadline = time.time() + timeout + workers = {} + while time.time() < deadline: + workers = cli.pools_launches(pool_name, host=host) + for worker in workers.values(): + if predicate(worker): + return worker + time.sleep(0.5) + raise TimeoutError(f"no worker matched within {timeout}s: {workers}") + + +class TestEcsLauncher: + def test_runs_worker_as_task(self, ecs_env): + """A worker is a task run from the pool's task definition, with the + modules as its command and the connection details as its environment, + in a signed request.""" + host = ecs_env["host"] + executor = ecs_env["executor"] + fake = ecs_env["ecs"] + targets = [workflow("test", "greet", parameters=["name"])] + _setup_ecs_pool(ecs_env, targets) + + resp = cli.submit("test/greet", '"world"', host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + assert ex.target == "greet" + assert ex.arguments[0]["value"] == "world" + ex.conn.complete(ex.execution_id, value="hello world") + + result = poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + assert result["type"] == "value" + assert result["value"]["data"] == "hello world" + + # Without a container named, the task definition says which to override. + assert len(fake.requests_for("DescribeTaskDefinition")) == 1 + + [(run_task, headers)] = fake.requests_for("RunTask") + assert run_task["cluster"] == fake.cluster + assert run_task["taskDefinition"] == "worker-task" + assert run_task["count"] == 1 + assert run_task["launchType"] == "FARGATE" + assert run_task["startedBy"] == "coflux:ecs-pool" + assert run_task["networkConfiguration"] == { + "awsvpcConfiguration": { + "subnets": ["subnet-1", "subnet-2"], + "securityGroups": ["sg-1"], + "assignPublicIp": "ENABLED", + } + } + [override] = run_task["overrides"]["containerOverrides"] + assert override["name"] == fake.container_name + assert override["command"] == ["test"] + env = {e["name"]: e["value"] for e in override["environment"]} + assert env["COFLUX_HOST"] == host + assert env["COFLUX_WORKSPACE"] == "default" + assert env["COFLUX_SESSION"] + + assert headers["content-type"] == "application/x-amz-json-1.1" + authorization = headers["authorization"] + assert authorization.startswith("AWS4-HMAC-SHA256 Credential=AKIATEST/") + assert "/us-east-1/ecs/aws4_request" in authorization + assert "x-amz-date" in headers + + def test_idle_worker_is_stopped(self, ecs_env): + """An idle worker's task is stopped, and a task stopped on request + isn't reported as having failed.""" + host = ecs_env["host"] + executor = ecs_env["executor"] + fake = ecs_env["ecs"] + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) + + resp = cli.submit("test/greet", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="done") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + [(stop_task, _)] = fake.wait_for("StopTask", timeout=30) + assert stop_task["cluster"] == fake.cluster + assert stop_task["task"] in fake.task_arns() + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["deactivatedAt"] is not None + ) + assert worker["stopError"] is None + assert worker["error"] is None + + def test_oom_killed_task_is_reported(self, ecs_env): + """A task that ECS stops for exceeding its memory is reported as + such, with the task's stopped reason in place of a log tail.""" + host = ecs_env["host"] + executor = ecs_env["executor"] + fake = ecs_env["ecs"] + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) + + cli.submit("test/greet", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + # Mid-execution, so the worker isn't idle and stopped first. + executor.next_execute(timeout=_EXEC_TIMEOUT) + + [arn] = fake.task_arns() + fake.kill_with_oom(arn) + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["deactivatedAt"] is not None, timeout=45 + ) + assert worker["error"] == "oom_killed" + assert worker["logs"] == "Essential container in task exited" + + def test_refused_launch_is_reported(self, ecs_env): + """A RunTask the API refuses fails the worker, with the API's own + message kept as the worker's logs.""" + host = ecs_env["host"] + fake = ecs_env["ecs"] + fake.run_task_error = ("ClusterNotFoundException", "Cluster not found.") + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) + + cli.submit("test/greet", host=host) + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["startError"] is not None + ) + assert worker["startError"] == "launch_cluster_not_found" + assert worker["logs"] == "Cluster not found." + + def test_missing_task_definition_is_reported(self, ecs_env): + """The API doesn't say a task definition wasn't found, only that it + couldn't be described; the worker says which it means.""" + host = ecs_env["host"] + fake = ecs_env["ecs"] + fake.task_definition_error = ( + "ClientException", + "Unable to describe task definition.", + ) + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) + + cli.submit("test/greet", host=host) + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["startError"] is not None + ) + assert worker["startError"] == "launch_task_definition_not_found" + assert worker["logs"] == "Unable to describe task definition." + + def test_capacity_provider_replaces_launch_type(self, ecs_env): + """A capacity provider is a strategy rather than a launch type, and a + named container isn't looked up.""" + host = ecs_env["host"] + executor = ecs_env["executor"] + fake = ecs_env["ecs"] + _setup_ecs_pool( + ecs_env, + [workflow("test", "greet")], + sets=["capacityProvider=FARGATE_SPOT", "containerName=app"], + ) + + resp = cli.submit("test/greet", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="done") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + assert fake.requests_for("DescribeTaskDefinition") == [] + [(run_task, _)] = fake.requests_for("RunTask") + assert "launchType" not in run_task + assert run_task["capacityProviderStrategy"] == [ + {"capacityProvider": "FARGATE_SPOT", "weight": 1} + ] + assert run_task["overrides"]["containerOverrides"][0]["name"] == "app" + + def test_launch_type_and_capacity_provider_are_exclusive(self, ecs_env): + with pytest.raises(subprocess.CalledProcessError): + _setup_ecs_pool( + ecs_env, + [workflow("test", "greet")], + sets=["launchType=EC2", "capacityProvider=FARGATE_SPOT"], + ) + + def test_single_ids_are_accepted_for_lists(self, ecs_env): + """A lone subnet or security group ID needn't be written as JSON.""" + host = ecs_env["host"] + _setup_ecs_pool(ecs_env, [workflow("test", "greet")], sets=["subnets=subnet-9"]) + + launcher = cli.pools_get("ecs-pool", host=host)["launcher"] + assert launcher["type"] == "ecs" + assert launcher["subnets"] == ["subnet-9"] + assert launcher["securityGroups"] == ["sg-1"] + assert launcher["assignPublicIp"] is True + # The key ID identifies the credentials; the secret stays out. + assert launcher["accessKeyId"] == "AKIATEST" + assert "secretAccessKey" not in launcher + + cli._coflux( + "pools", + "update", + "ecs-pool", + "--set", + "subnets=subnet-10", + host=host, + output=None, + ) + assert cli.pools_get("ecs-pool", host=host)["launcher"]["subnets"] == [ + "subnet-10" + ] + + def test_export_redacts_credentials(self, ecs_env, tmp_path): + """The secret key and session token are secrets; the key ID isn't.""" + host = ecs_env["host"] + _setup_ecs_pool( + ecs_env, + [workflow("test", "greet")], + sets=["sessionToken=test-session-token"], + ) + + exported = cli.pools_export(host=host) + assert "test-secret-key" not in exported + assert "test-session-token" not in exported + assert 'secret_access_key = ""' in exported + assert 'session_token = ""' in exported + assert 'access_key_id = "AKIATEST"' in exported + assert 'task_definition = "worker-task"' in exported + + path = tmp_path / "pools.toml" + path.write_text(exported) + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.pools_import(path, host=host) + assert "ecs-pool" in exc_info.value.stderr + + with_secrets = cli.pools_export(include_secrets=True, host=host) + assert "test-secret-key" in with_secrets + assert "test-session-token" in with_secrets + path.write_text(with_secrets) + cli.pools_import(path, host=host) + assert "test-secret-key" in cli.pools_export(include_secrets=True, host=host) From 65e8a6003deb2ec95039c754aa12185c0b838cfe Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 20:10:32 +0100 Subject: [PATCH 03/21] Support configuring pool idle timeout --- cli/CHANGELOG.md | 1 + cli/cmd/coflux/pools.go | 16 ++++--- docs/docs/pools.md | 1 + server/lib/coflux/handlers/api.ex | 12 ++++++ .../coflux/orchestration/server/scheduler.ex | 21 +++++++--- server/lib/coflux/topics/pool.ex | 1 + tests/support/cli.py | 7 ++++ tests/test_pools.py | 42 +++++++++++++++++++ 8 files changed, 91 insertions(+), 10 deletions(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 34ad6bee..ea1239c9 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -3,6 +3,7 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`. +- Adds the `idleTimeout` launcher field, for how long a pool keeps an idle worker before stopping it. ## 0.12.0 diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 0ef6c4c4..497ee248 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -228,6 +228,9 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if concurrency := getFloat64(launcher, "concurrency"); concurrency > 0 { fmt.Printf("Concurrency: %d\n", int(concurrency)) } + if idleTimeout, ok := launcher["idleTimeout"].(float64); ok { + fmt.Printf("Idle timeout: %ds\n", int(idleTimeout)) + } if env, ok := launcher["env"].(map[string]any); ok && len(env) > 0 { fmt.Printf("Environment:\n") for k, v := range env { @@ -633,7 +636,7 @@ var launcherFields = map[string]bool{ "platformVersion": true, "accessKeyId": true, "secretAccessKey": true, "sessionToken": true, "endpoint": true, "serverHost": true, "serverSecure": true, "adapter": true, - "concurrency": true, "env": true, + "concurrency": true, "idleTimeout": true, "env": true, } // mapSubkeyFields lists launcher fields that support dotted sub-key access @@ -1432,6 +1435,7 @@ var camelToSnake = map[string]string{ "networkMode": "network_mode", "serverHost": "server_host", "serverSecure": "server_secure", + "idleTimeout": "idle_timeout", "serviceAccount": "service_account", "apiServer": "api_server", "imagePullPolicy": "image_pull_policy", @@ -1533,10 +1537,12 @@ func tomlLauncherToAPI(launcher map[string]any) map[string]any { result[k] = v } } - // Ensure concurrency is an integer (TOML int64 → JSON number) - if c, ok := result["concurrency"]; ok { - if i, ok := c.(int64); ok { - result["concurrency"] = int(i) + // Ensure integer fields are integers (TOML int64 → JSON number) + for _, key := range []string{"concurrency", "idleTimeout"} { + if v, ok := result[key]; ok { + if i, ok := v.(int64); ok { + result[key] = int(i) + } } } return result diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 92767493..d3c1f305 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -176,6 +176,7 @@ These fields apply to all launcher types: | `serverSecure` | Use TLS for server connection | | `adapter` | Adapter command | | `concurrency` | Maximum concurrent executions per worker | +| `idleTimeout` | Seconds an idle worker is kept before being stopped (default: 5). Worth raising for launchers with slow starts, such as ECS | | `env` | Environment variables (e.g., `--set env.KEY=VALUE`) | ## Managing pools diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 14711876..6dbd6234 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -1488,6 +1488,7 @@ defmodule Coflux.Handlers.Api do server_secure = Map.get(value, "serverSecure") adapter = Map.get(value, "adapter") concurrency = Map.get(value, "concurrency") + idle_timeout = Map.get(value, "idleTimeout") env = Map.get(value, "env") cond do @@ -1505,6 +1506,9 @@ defmodule Coflux.Handlers.Api do not is_nil(concurrency) and (not is_integer(concurrency) or concurrency < 1) -> {:error, :invalid} + not is_nil(idle_timeout) and (not is_integer(idle_timeout) or idle_timeout < 0) -> + {:error, :invalid} + not is_nil(env) and not is_map(env) -> {:error, :invalid} @@ -1528,6 +1532,11 @@ defmodule Coflux.Handlers.Api do launcher = if concurrency, do: Map.put(launcher, :concurrency, concurrency), else: launcher + launcher = + if not is_nil(idle_timeout), + do: Map.put(launcher, :idle_timeout, idle_timeout), + else: launcher + launcher = if env, do: Map.put(launcher, :env, env), else: launcher {:ok, launcher} end @@ -1683,6 +1692,7 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("serverSecure", Map.get(launcher, :server_secure)) |> maybe_put_value("adapter", Map.get(launcher, :adapter)) |> maybe_put_value("concurrency", Map.get(launcher, :concurrency)) + |> maybe_put_value("idleTimeout", Map.get(launcher, :idle_timeout)) |> maybe_put_value("env", Map.get(launcher, :env)) end @@ -1830,6 +1840,7 @@ defmodule Coflux.Handlers.Api do {"serverSecure", &is_boolean/1}, {"adapter", fn v -> is_list(v) and v != [] and Enum.all?(v, &is_binary/1) end}, {"concurrency", fn v -> is_integer(v) and v >= 1 end}, + {"idleTimeout", fn v -> is_integer(v) and v >= 0 end}, {"env", fn v -> is_map(v) and @@ -1876,6 +1887,7 @@ defmodule Coflux.Handlers.Api do "serverSecure" => :server_secure, "adapter" => :adapter, "concurrency" => :concurrency, + "idleTimeout" => :idle_timeout, "env" => :env } diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 345c2cb4..a35ee1b8 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -58,10 +58,11 @@ defmodule Coflux.Orchestration.Server.Scheduler do @connected_worker_poll_interval_ms 30_000 @disconnected_worker_poll_interval_ms 5_000 - # How long a *ready* worker sits idle before it is stopped. A worker - # that has never declared targets is not idle, it is still starting - - # see `Fleet.session_ready?/1`. - @worker_idle_timeout_ms 5_000 + # How long a *ready* worker sits idle before it is stopped, unless its + # pool says otherwise (see `worker_idle_timeout_ms/2`). A worker that + # has never declared targets is not idle, it is still starting - see + # `Fleet.session_ready?/1`. + @default_worker_idle_timeout_ms 5_000 # How often to sweep while any worker exists, for the deadlines above. @sweep_interval_ms 5_000 @@ -679,7 +680,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do # time, but it means nothing. true <- Fleet.session_ready?(session) do Enum.empty?(session.starting) && Enum.empty?(session.executing) && - now - session.last_idle_at >= @worker_idle_timeout_ms + now - session.last_idle_at >= worker_idle_timeout_ms(state, worker) else _ -> false end @@ -822,6 +823,16 @@ defmodule Coflux.Orchestration.Server.Scheduler do ) end + # A pool can say how long its workers linger once idle, in seconds: a + # worker that takes a while to start is worth keeping warm between + # runs. One whose pool doesn't say gets the default. + defp worker_idle_timeout_ms(state, worker) do + case get_in(state.pools, [worker.workspace_id, worker.pool_name, :launcher, :idle_timeout]) do + seconds when is_integer(seconds) and seconds >= 0 -> seconds * 1000 + _ -> @default_worker_idle_timeout_ms + end + end + defp poll_due?(state, worker, now) do cond do is_nil(worker.data) -> diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index d8afa1bc..18ec9ff2 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -110,6 +110,7 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:serverSecure, Map.get(launcher, :server_secure)) |> maybe_put(:adapter, Map.get(launcher, :adapter)) |> maybe_put(:concurrency, Map.get(launcher, :concurrency)) + |> maybe_put(:idleTimeout, Map.get(launcher, :idle_timeout)) |> maybe_put(:env, Map.get(launcher, :env)) end diff --git a/tests/support/cli.py b/tests/support/cli.py index d9de68dc..ecfe495c 100644 --- a/tests/support/cli.py +++ b/tests/support/cli.py @@ -262,6 +262,7 @@ def _pools_set_args( docker_image=None, adapter=None, concurrency=None, + idle_timeout=None, env=None, ): """Build --set/--modules/--provides/--accepts args for pool create/update.""" @@ -282,6 +283,8 @@ def _pools_set_args( args.extend(["--set", f"adapter={json.dumps(adapter)}"]) if concurrency: args.extend(["--set", f"concurrency={concurrency}"]) + if idle_timeout is not None: + args.extend(["--set", f"idleTimeout={idle_timeout}"]) if env: for k, v in env.items(): args.extend(["--set", f"env.{k}={v}"]) @@ -298,6 +301,7 @@ def pools_create( docker_image=None, adapter=None, concurrency=None, + idle_timeout=None, env=None, host=None, workspace="default", @@ -312,6 +316,7 @@ def pools_create( docker_image=docker_image, adapter=adapter, concurrency=concurrency, + idle_timeout=idle_timeout, env=env, ) ) @@ -327,6 +332,7 @@ def pools_update( docker_image=None, adapter=None, concurrency=None, + idle_timeout=None, env=None, host=None, workspace="default", @@ -341,6 +347,7 @@ def pools_update( docker_image=docker_image, adapter=adapter, concurrency=concurrency, + idle_timeout=idle_timeout, env=env, ) ) diff --git a/tests/test_pools.py b/tests/test_pools.py index 983a4b59..690ca5a7 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -296,6 +296,48 @@ def test_get_returns_env(self, pool_env): assert pool["launcher"]["env"]["MY_VAR"] == "hello" assert pool["launcher"]["env"]["OTHER_VAR"] == "world" + def test_get_returns_idle_timeout(self, pool_env): + """An idle timeout configured on a pool is returned in pool details, + including zero.""" + host = pool_env["host"] + targets = [workflow("test", "my_workflow")] + _setup_pool(pool_env, targets, pool_name="idle-pool", idle_timeout=300) + + assert cli.pools_get("idle-pool", host=host)["launcher"]["idleTimeout"] == 300 + + cli.pools_update("idle-pool", idle_timeout=0, host=host) + assert cli.pools_get("idle-pool", host=host)["launcher"]["idleTimeout"] == 0 + + def test_idle_timeout_keeps_worker_warm(self, pool_env): + """A worker with an idle timeout outlives the gap between runs, so + the second run reuses it rather than paying for another launch.""" + host = pool_env["host"] + executor = pool_env["executor"] + targets = [workflow("test", "greet", parameters=["name"])] + _setup_pool(pool_env, targets, pool_name="warm-pool", idle_timeout=60) + + resp = cli.submit("test/greet", '"one"', host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="one") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + # Longer than the default idle timeout and the sweep that enforces it. + time.sleep(12) + workers = cli.pools_launches("warm-pool", host=host) + assert len(workers) == 1 + assert all(w["stoppingAt"] is None for w in workers.values()) + + resp = cli.submit("test/greet", '"two"', host=host) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="two") + result = poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + assert result["value"]["data"] == "two" + + # Same worker, same connection: nothing else was launched. + executor.wait_connections(1, timeout=1) + assert len(cli.pools_launches("warm-pool", host=host)) == 1 + def test_update_common_fields(self, pool_env): """Common launcher fields can be updated on an existing pool.""" host = pool_env["host"] From 289dec45784bc3a80df41591bd46b2c7ffd38f9e Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 20:29:51 +0100 Subject: [PATCH 04/21] Support configuring idle timeout --- cli/CHANGELOG.md | 2 +- cli/cmd/coflux/pools.go | 39 ++++++++++++------- docs/docs/pools.md | 2 +- server/lib/coflux/handlers/api.ex | 19 ++++----- server/lib/coflux/orchestration/epoch.ex | 7 ++-- .../coflux/orchestration/server/scheduler.ex | 2 +- server/lib/coflux/orchestration/workspaces.ex | 37 ++++++++++++++---- server/lib/coflux/topics/pool.ex | 2 +- server/priv/migrations/orchestration/6.sql | 4 ++ tests/test_pools.py | 31 ++++++++++++--- 10 files changed, 100 insertions(+), 45 deletions(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index ea1239c9..ad6616ae 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -3,7 +3,7 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`. -- Adds the `idleTimeout` launcher field, for how long a pool keeps an idle worker before stopping it. +- Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it. ## 0.12.0 diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 497ee248..e5218e46 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -155,6 +155,11 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { fmt.Printf("Accepts: %s\n", accepts) } + // Idle timeout + if idleTimeout, ok := pool["idleTimeout"].(float64); ok { + fmt.Printf("Idle timeout: %ds\n", int(idleTimeout)) + } + // Launcher if launcher, ok := pool["launcher"].(map[string]any); ok { fmt.Printf("Launcher: %s\n", getString(launcher, "type")) @@ -228,9 +233,6 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if concurrency := getFloat64(launcher, "concurrency"); concurrency > 0 { fmt.Printf("Concurrency: %d\n", int(concurrency)) } - if idleTimeout, ok := launcher["idleTimeout"].(float64); ok { - fmt.Printf("Idle timeout: %ds\n", int(idleTimeout)) - } if env, ok := launcher["env"].(map[string]any); ok && len(env) > 0 { fmt.Printf("Environment:\n") for k, v := range env { @@ -616,9 +618,10 @@ func parseSetValue(s string) any { // poolTopLevelFields lists field names that are pool-level (not launcher-level). var poolTopLevelFields = map[string]bool{ - "modules": true, - "provides": true, - "accepts": true, + "modules": true, + "provides": true, + "accepts": true, + "idleTimeout": true, } // launcherFields lists valid launcher field names. @@ -636,7 +639,7 @@ var launcherFields = map[string]bool{ "platformVersion": true, "accessKeyId": true, "secretAccessKey": true, "sessionToken": true, "endpoint": true, "serverHost": true, "serverSecure": true, "adapter": true, - "concurrency": true, "idleTimeout": true, "env": true, + "concurrency": true, "env": true, } // mapSubkeyFields lists launcher fields that support dotted sub-key access @@ -1435,7 +1438,6 @@ var camelToSnake = map[string]string{ "networkMode": "network_mode", "serverHost": "server_host", "serverSecure": "server_secure", - "idleTimeout": "idle_timeout", "serviceAccount": "service_account", "apiServer": "api_server", "imagePullPolicy": "image_pull_policy", @@ -1486,6 +1488,9 @@ func apiPoolToTOML(pool map[string]any) map[string]any { result["accepts"] = accepts } } + if idleTimeout, ok := pool["idleTimeout"]; ok { + result["idle_timeout"] = idleTimeout + } if launcher, ok := pool["launcher"].(map[string]any); ok { result["launcher"] = apiLauncherToTOML(launcher) } @@ -1522,6 +1527,14 @@ func tomlPoolToAPI(pool map[string]any) map[string]any { if accepts, ok := pool["accepts"]; ok { result["accepts"] = toStringSliceMap(accepts) } + if idleTimeout, ok := pool["idle_timeout"]; ok { + // TOML int64 → JSON number + if i, ok := idleTimeout.(int64); ok { + result["idleTimeout"] = int(i) + } else { + result["idleTimeout"] = idleTimeout + } + } if launcher, ok := pool["launcher"].(map[string]any); ok { result["launcher"] = tomlLauncherToAPI(launcher) } @@ -1537,12 +1550,10 @@ func tomlLauncherToAPI(launcher map[string]any) map[string]any { result[k] = v } } - // Ensure integer fields are integers (TOML int64 → JSON number) - for _, key := range []string{"concurrency", "idleTimeout"} { - if v, ok := result[key]; ok { - if i, ok := v.(int64); ok { - result[key] = int(i) - } + // Ensure concurrency is an integer (TOML int64 → JSON number) + if c, ok := result["concurrency"]; ok { + if i, ok := c.(int64); ok { + result["concurrency"] = int(i) } } return result diff --git a/docs/docs/pools.md b/docs/docs/pools.md index d3c1f305..9e15e236 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -172,11 +172,11 @@ These fields apply to all launcher types: | `--modules`, `-m` | Modules to host (can be specified multiple times). Module names only — wildcards aren't supported, since this is also what launched workers are told to import | | `--provides` | Features that workers provide (e.g., `gpu:A100`) | | `--accepts` | Tags that executions must have to be assigned to this pool | +| `idleTimeout` | Seconds the pool keeps an idle worker before stopping it (default: 5). Worth raising for launchers with slow starts, such as ECS | | `serverHost` | Server host override for launched workers | | `serverSecure` | Use TLS for server connection | | `adapter` | Adapter command | | `concurrency` | Maximum concurrent executions per worker | -| `idleTimeout` | Seconds an idle worker is kept before being stopped (default: 5). Worth raising for launchers with slow starts, such as ECS | | `env` | Environment variables (e.g., `--set env.KEY=VALUE`) | ## Managing pools diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 6dbd6234..34199c23 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -1488,7 +1488,6 @@ defmodule Coflux.Handlers.Api do server_secure = Map.get(value, "serverSecure") adapter = Map.get(value, "adapter") concurrency = Map.get(value, "concurrency") - idle_timeout = Map.get(value, "idleTimeout") env = Map.get(value, "env") cond do @@ -1506,9 +1505,6 @@ defmodule Coflux.Handlers.Api do not is_nil(concurrency) and (not is_integer(concurrency) or concurrency < 1) -> {:error, :invalid} - not is_nil(idle_timeout) and (not is_integer(idle_timeout) or idle_timeout < 0) -> - {:error, :invalid} - not is_nil(env) and not is_map(env) -> {:error, :invalid} @@ -1532,11 +1528,6 @@ defmodule Coflux.Handlers.Api do launcher = if concurrency, do: Map.put(launcher, :concurrency, concurrency), else: launcher - launcher = - if not is_nil(idle_timeout), - do: Map.put(launcher, :idle_timeout, idle_timeout), - else: launcher - launcher = if env, do: Map.put(launcher, :env, env), else: launcher {:ok, launcher} end @@ -1622,6 +1613,7 @@ defmodule Coflux.Handlers.Api do config = if Enum.any?(provides), do: Map.put(config, "provides", provides), else: config config = if Enum.any?(accepts), do: Map.put(config, "accepts", accepts), else: config + config = maybe_put_value(config, "idleTimeout", Map.get(pool, :idle_timeout)) if pool.launcher do Map.put(config, "launcher", build_launcher_config(pool.launcher, include_secrets)) @@ -1692,7 +1684,6 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("serverSecure", Map.get(launcher, :server_secure)) |> maybe_put_value("adapter", Map.get(launcher, :adapter)) |> maybe_put_value("concurrency", Map.get(launcher, :concurrency)) - |> maybe_put_value("idleTimeout", Map.get(launcher, :idle_timeout)) |> maybe_put_value("env", Map.get(launcher, :env)) end @@ -1707,6 +1698,7 @@ defmodule Coflux.Handlers.Api do {"modules", &parse_modules/1, :modules, []}, {"provides", &parse_tag_set/1, :provides, %{}}, {"accepts", &parse_tag_set/1, :accepts, %{}}, + {"idleTimeout", &parse_idle_timeout/1, :idle_timeout, nil}, {"launcher", &parse_launcher/1, :launcher, nil} ], {:ok, %{}}, @@ -1735,6 +1727,10 @@ defmodule Coflux.Handlers.Api do end end + # Seconds an idle worker is kept for. Zero means the next sweep. + defp parse_idle_timeout(value) when is_integer(value) and value >= 0, do: {:ok, value} + defp parse_idle_timeout(_value), do: {:error, :invalid} + # Parses a partial pool update (PATCH semantics). # Only keys present in the JSON are included. A JSON null value means "unset". defp parse_pool_patch(value) do @@ -1744,6 +1740,7 @@ defmodule Coflux.Handlers.Api do {"modules", &parse_modules/1, :modules}, {"provides", &parse_tag_set/1, :provides}, {"accepts", &parse_tag_set/1, :accepts}, + {"idleTimeout", &parse_idle_timeout/1, :idle_timeout}, {"launcher", &parse_launcher_patch/1, :launcher} ] @@ -1840,7 +1837,6 @@ defmodule Coflux.Handlers.Api do {"serverSecure", &is_boolean/1}, {"adapter", fn v -> is_list(v) and v != [] and Enum.all?(v, &is_binary/1) end}, {"concurrency", fn v -> is_integer(v) and v >= 1 end}, - {"idleTimeout", fn v -> is_integer(v) and v >= 0 end}, {"env", fn v -> is_map(v) and @@ -1887,7 +1883,6 @@ defmodule Coflux.Handlers.Api do "serverSecure" => :server_secure, "adapter" => :adapter, "concurrency" => :concurrency, - "idleTimeout" => :idle_timeout, "env" => :env } diff --git a/server/lib/coflux/orchestration/epoch.ex b/server/lib/coflux/orchestration/epoch.ex index 19630577..8677ca10 100644 --- a/server/lib/coflux/orchestration/epoch.ex +++ b/server/lib/coflux/orchestration/epoch.ex @@ -1979,10 +1979,10 @@ defmodule Coflux.Orchestration.Epoch do defp ensure_pool_definition(_source_db, _target_db, nil), do: nil defp ensure_pool_definition(source_db, target_db, old_id) do - {:ok, {hash, launcher_id, provides_tag_set_id, accepts_tag_set_id}} = + {:ok, {hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout}} = query_one!( source_db, - "SELECT hash, launcher_id, provides_tag_set_id, accepts_tag_set_id FROM pool_definitions WHERE id = ?1", + "SELECT hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout FROM pool_definitions WHERE id = ?1", {old_id} ) @@ -2000,7 +2000,8 @@ defmodule Coflux.Orchestration.Epoch do hash: {:blob, hash}, launcher_id: ensure_launcher(source_db, target_db, launcher_id), provides_tag_set_id: ensure_tag_set(source_db, target_db, provides_tag_set_id), - accepts_tag_set_id: ensure_tag_set(source_db, target_db, accepts_tag_set_id) + accepts_tag_set_id: ensure_tag_set(source_db, target_db, accepts_tag_set_id), + idle_timeout: idle_timeout }) # Copy pool_definition_modules diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index a35ee1b8..a2b94cac 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -827,7 +827,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do # worker that takes a while to start is worth keeping warm between # runs. One whose pool doesn't say gets the default. defp worker_idle_timeout_ms(state, worker) do - case get_in(state.pools, [worker.workspace_id, worker.pool_name, :launcher, :idle_timeout]) do + case get_in(state.pools, [worker.workspace_id, worker.pool_name, :idle_timeout]) do seconds when is_integer(seconds) and seconds >= 0 -> seconds * 1000 _ -> @default_worker_idle_timeout_ms end diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index c2429ade..1ee5fb39 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -547,6 +547,7 @@ defmodule Coflux.Orchestration.Workspaces do |> apply_patch_field(patch, :modules) |> apply_patch_field(patch, :provides) |> apply_patch_field(patch, :accepts) + |> apply_patch_field(patch, :idle_timeout) |> apply_launcher_patch(patch) end @@ -699,7 +700,14 @@ defmodule Coflux.Orchestration.Workspaces do end end - defp hash_pool_definition(db, launcher_id, provides_tag_set_id, accepts_tag_set_id, modules) do + defp hash_pool_definition( + db, + launcher_id, + provides_tag_set_id, + accepts_tag_set_id, + modules, + idle_timeout + ) do launcher_hash = if launcher_id do {:ok, {hash}} = @@ -730,6 +738,10 @@ defmodule Coflux.Orchestration.Workspaces do <<0>> end + # Only part of the hash when set, so a definition without one keeps + # the hash it had before the field existed. + idle_timeout_part = if idle_timeout, do: [Integer.to_string(idle_timeout)], else: [] + data = Enum.intersperse( [ @@ -737,7 +749,7 @@ defmodule Coflux.Orchestration.Workspaces do tag_set_hash, accepts_tag_set_hash, Enum.join(Enum.sort(modules), "\n") - ], + ] ++ idle_timeout_part, 0 ) @@ -749,6 +761,7 @@ defmodule Coflux.Orchestration.Workspaces do provides = Map.get(pool, :provides, %{}) accepts = Map.get(pool, :accepts, %{}) launcher = Map.get(pool, :launcher) + idle_timeout = Map.get(pool, :idle_timeout) launcher_id = if launcher do @@ -771,7 +784,15 @@ defmodule Coflux.Orchestration.Workspaces do end end - hash = hash_pool_definition(db, launcher_id, provides_tag_set_id, accepts_tag_set_id, modules) + hash = + hash_pool_definition( + db, + launcher_id, + provides_tag_set_id, + accepts_tag_set_id, + modules, + idle_timeout + ) case query_one(db, "SELECT id FROM pool_definitions WHERE hash = ?1", {{:blob, hash}}) do {:ok, {id}} -> @@ -783,7 +804,8 @@ defmodule Coflux.Orchestration.Workspaces do hash: {:blob, hash}, provides_tag_set_id: provides_tag_set_id, accepts_tag_set_id: accepts_tag_set_id, - launcher_id: launcher_id + launcher_id: launcher_id, + idle_timeout: idle_timeout }) {:ok, _} = @@ -864,10 +886,10 @@ defmodule Coflux.Orchestration.Workspaces do defp get_pool_definition(db, pool_definition_id) do case query_one( db, - "SELECT launcher_id, provides_tag_set_id, accepts_tag_set_id FROM pool_definitions WHERE id = ?1", + "SELECT launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout FROM pool_definitions WHERE id = ?1", {pool_definition_id} ) do - {:ok, {launcher_id, provides_tag_set_id, accepts_tag_set_id}} -> + {:ok, {launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout}} -> provides = if provides_tag_set_id do case TagSets.get_tag_set(db, provides_tag_set_id) do @@ -908,7 +930,8 @@ defmodule Coflux.Orchestration.Workspaces do provides: provides, accepts: accepts, modules: modules, - launcher: launcher + launcher: launcher, + idle_timeout: idle_timeout }} {:ok, nil} -> diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index 18ec9ff2..a88f68c5 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -46,6 +46,7 @@ defmodule Coflux.Topics.Pool do launcher: if(pool.launcher, do: build_launcher(pool.launcher)), state: to_string(Map.get(pool, :state, :active)) } + |> maybe_put(:idleTimeout, Map.get(pool, :idle_timeout)) end defp build_launcher(launcher) do @@ -110,7 +111,6 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:serverSecure, Map.get(launcher, :server_secure)) |> maybe_put(:adapter, Map.get(launcher, :adapter)) |> maybe_put(:concurrency, Map.get(launcher, :concurrency)) - |> maybe_put(:idleTimeout, Map.get(launcher, :idle_timeout)) |> maybe_put(:env, Map.get(launcher, :env)) end diff --git a/server/priv/migrations/orchestration/6.sql b/server/priv/migrations/orchestration/6.sql index 5e270b89..8d4a90c4 100644 --- a/server/priv/migrations/orchestration/6.sql +++ b/server/priv/migrations/orchestration/6.sql @@ -134,3 +134,7 @@ ALTER TABLE assignments ADD COLUMN catalog_sequence INTEGER; -- NULL means "not chosen": resolve by the rule above. ALTER TABLE runs ADD COLUMN catalog_sequence INTEGER; ALTER TABLE executions ADD COLUMN catalog_sequence INTEGER; + +-- How long a pool keeps an idle worker before stopping it, in seconds. +-- NULL leaves it to the scheduler's default. +ALTER TABLE pool_definitions ADD COLUMN idle_timeout INTEGER; diff --git a/tests/test_pools.py b/tests/test_pools.py index 690ca5a7..c5399ea0 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -296,17 +296,38 @@ def test_get_returns_env(self, pool_env): assert pool["launcher"]["env"]["MY_VAR"] == "hello" assert pool["launcher"]["env"]["OTHER_VAR"] == "world" - def test_get_returns_idle_timeout(self, pool_env): - """An idle timeout configured on a pool is returned in pool details, - including zero.""" + def test_idle_timeout_is_a_pool_field(self, pool_env, tmp_path): + """An idle timeout is set, exported, unset and imported at the pool + level, and zero is a value rather than an absence.""" host = pool_env["host"] targets = [workflow("test", "my_workflow")] _setup_pool(pool_env, targets, pool_name="idle-pool", idle_timeout=300) - assert cli.pools_get("idle-pool", host=host)["launcher"]["idleTimeout"] == 300 + pool = cli.pools_get("idle-pool", host=host) + assert pool["idleTimeout"] == 300 + assert "idleTimeout" not in pool["launcher"] + + exported = cli.pools_export(host=host) + assert "idle_timeout = 300" in exported cli.pools_update("idle-pool", idle_timeout=0, host=host) - assert cli.pools_get("idle-pool", host=host)["launcher"]["idleTimeout"] == 0 + assert cli.pools_get("idle-pool", host=host)["idleTimeout"] == 0 + + cli._coflux( + "pools", + "update", + "idle-pool", + "--unset", + "idleTimeout", + host=host, + output=None, + ) + assert "idleTimeout" not in cli.pools_get("idle-pool", host=host) + + path = tmp_path / "pools.toml" + path.write_text(exported) + cli.pools_import(path, host=host) + assert cli.pools_get("idle-pool", host=host)["idleTimeout"] == 300 def test_idle_timeout_keeps_worker_warm(self, pool_env): """A worker with an idle timeout outlives the gap between runs, so From a62b8569ebf71efc4937971c270ad0dac05d3fb4 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 21:22:44 +0100 Subject: [PATCH 05/21] Store tokens in admin database --- docs/docs/server_config.md | 2 +- server/lib/coflux/admin/tokens.ex | 230 +++++++++++++++++ server/lib/coflux/orchestration/epoch.ex | 72 +----- server/lib/coflux/orchestration/principals.ex | 244 +++--------------- server/lib/coflux/orchestration/results.ex | 6 +- server/lib/coflux/orchestration/runs.ex | 15 +- server/lib/coflux/orchestration/server.ex | 50 +++- .../coflux/orchestration/server/archives.ex | 4 +- .../coflux/orchestration/server/rotation.ex | 4 +- .../coflux/orchestration/server/snapshots.ex | 6 +- .../lib/coflux/orchestration/server/state.ex | 4 + server/lib/coflux/store/epochs.ex | 15 ++ server/lib/coflux/store/migrations.ex | 35 ++- server/priv/migrations/admin/1.sql | 19 ++ server/priv/migrations/orchestration/6.sql | 22 ++ server/test/coflux/admin_tokens_test.exs | 115 +++++++++ server/test/coflux/migrations_test.exs | 8 + tests/test_tokens.py | 109 ++++++++ 18 files changed, 645 insertions(+), 315 deletions(-) create mode 100644 server/lib/coflux/admin/tokens.ex create mode 100644 server/priv/migrations/admin/1.sql create mode 100644 server/test/coflux/admin_tokens_test.exs create mode 100644 tests/test_tokens.py diff --git a/docs/docs/server_config.md b/docs/docs/server_config.md index 3da68066..67e7ccc8 100644 --- a/docs/docs/server_config.md +++ b/docs/docs/server_config.md @@ -75,4 +75,4 @@ The server is configured via environment variables. When using `coflux server`, ## Data storage -The server stores data in the configured data directory. Each project gets its own SQLite database. Data is managed in rotating epochs, which allows the server to manage data growth without losing access to historical runs. +The server stores data in the configured data directory. Each project gets its own set of SQLite databases. Orchestration, logs and metrics data is managed in rotating epochs, which allows the server to manage data growth without losing access to historical runs. An admin database holds service tokens, and isn't rotated. diff --git a/server/lib/coflux/admin/tokens.ex b/server/lib/coflux/admin/tokens.ex new file mode 100644 index 00000000..94f4c8fc --- /dev/null +++ b/server/lib/coflux/admin/tokens.ex @@ -0,0 +1,230 @@ +defmodule Coflux.Admin.Tokens do + @moduledoc """ + Service tokens, in the admin store. + + The admin store isn't rotated, so a token is created once and found for + as long as it lives, whichever epoch is active. What a token *does* is + attributed in the orchestration store, through a principal that names + the token by external id (`Coflux.Orchestration.Principals`). + + Who created a token is kept here as an identity - a type and external + id - rather than a principal id, since principal ids are local to an + epoch. + """ + + alias Coflux.Store + alias Exqlite.Sqlite3 + + @doc """ + Validates a token by hash. + + Returns `{:ok, %{external_id: id, workspaces: patterns | nil}}`, with + `nil` workspaces meaning all of them, or `{:error, :not_found}`. + """ + def check_token(db, token_hash) do + now = System.system_time(:second) + + query = """ + SELECT external_id, workspaces + FROM tokens + WHERE token_hash = ?1 + AND revoked_at IS NULL + AND (expires_at IS NULL OR expires_at > ?2) + """ + + case Store.query_one(db, query, {token_hash, now}) do + {:ok, {external_id, workspaces_json}} -> + {:ok, %{external_id: external_id, workspaces: decode_workspaces(workspaces_json)}} + + {:ok, nil} -> + {:error, :not_found} + end + end + + @doc """ + Creates a token. `created_by` is the creator's identity as + `Principals.build/2` describes it, or nil. + + Options: + - workspaces: list of workspace patterns, or nil for all workspaces + """ + def create_token(db, project_id, name, created_by, opts \\ []) do + now = System.system_time(:second) + workspaces = Keyword.get(opts, :workspaces) + + # Generate random bytes (16 bytes = 32 hex chars) + random = :crypto.strong_rand_bytes(16) + random_hex = Base.encode16(random, case: :lower) + + # Compute signature (first 4 bytes of HMAC) + signature_hex = Coflux.Auth.compute_token_signature(random_hex, project_id) + + # Full token: cflx_ + token = "cflx_" <> random_hex <> signature_hex + + # Store hash of random part for lookup + random_hash = :crypto.hash(:sha256, random_hex) |> Base.encode16(case: :lower) + + {:ok, external_id} = Store.generate_external_id(db, "tokens", 12) + + {:ok, token_id} = + Store.insert_one(db, :tokens, %{ + external_id: external_id, + token_hash: random_hash, + name: name, + workspaces: encode_workspaces(workspaces), + created_by_type: created_by && created_by.type, + created_by_external_id: created_by && created_by.external_id, + created_at: now + }) + + {:ok, + %{ + id: token_id, + token: token, + token_id: token_id, + external_id: external_id, + name: name, + workspaces: workspaces, + created_at: now, + expires_at: nil, + revoked_at: nil, + created_by: created_by + }} + end + + @doc "Lists all tokens, revoked ones included, newest first." + def list_tokens(db) do + query = """ + SELECT id, external_id, name, workspaces, created_at, expires_at, revoked_at, + created_by_type, created_by_external_id + FROM tokens + ORDER BY created_at DESC + """ + + {:ok, rows} = Store.query(db, query, {}) + + {:ok, Enum.map(rows, &build_token/1)} + end + + @doc """ + Gets a token by its external_id. + Returns {:ok, token} or {:ok, nil} if not found. + """ + def get_token_by_external_id(db, external_id) do + query = """ + SELECT id, external_id, name, workspaces, created_at, expires_at, revoked_at, + created_by_type, created_by_external_id + FROM tokens + WHERE external_id = ?1 + """ + + case Store.query_one(db, query, {external_id}) do + {:ok, nil} -> {:ok, nil} + {:ok, row} -> {:ok, build_token(row)} + end + end + + @doc """ + Revokes a token by ID. + """ + def revoke_token(db, token_id) do + now = System.system_time(:second) + + case Store.query_one(db, "SELECT id, external_id FROM tokens WHERE id = ?1", {token_id}) do + {:ok, {^token_id, external_id}} -> + {:ok, _} = + Store.query(db, "UPDATE tokens SET revoked_at = ?1 WHERE id = ?2", {now, token_id}) + + {:ok, external_id} + + {:ok, nil} -> + {:error, :not_found} + end + end + + @doc """ + Moves tokens out of an orchestration database that still has them - one + from before the admin store existed - and drops the table. Nothing to do + when the table is already gone. Safe to repeat: a token that is already + in the admin store is left as it is. + + Runs after the orchestration migrations, which is what leaves the + principals there naming tokens by external id. + """ + def import_legacy(orchestration_db, admin_db) do + case Store.query_one( + orchestration_db, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'tokens'", + {} + ) do + {:ok, nil} -> + :ok + + {:ok, _} -> + {:ok, rows} = + Store.query( + orchestration_db, + """ + SELECT t.id, t.external_id, t.token_hash, t.name, t.workspaces, + t.created_at, t.expires_at, t.revoked_at, + p.user_external_id, p.token_external_id + FROM tokens AS t + LEFT JOIN principals AS p ON p.id = t.created_by + """, + {} + ) + + values = + Enum.map(rows, fn {id, external_id, token_hash, name, workspaces, created_at, + expires_at, revoked_at, creator_user_id, creator_token_id} -> + {type, creator_id} = + cond do + creator_user_id -> {"user", creator_user_id} + creator_token_id -> {"token", creator_token_id} + true -> {nil, nil} + end + + {id, external_id, token_hash, name, workspaces, type, creator_id, created_at, + expires_at, revoked_at} + end) + + {:ok, _} = + Store.insert_many( + admin_db, + :tokens, + {:id, :external_id, :token_hash, :name, :workspaces, :created_by_type, + :created_by_external_id, :created_at, :expires_at, :revoked_at}, + values, + on_conflict: "(external_id) DO NOTHING" + ) + + :ok = Sqlite3.execute(orchestration_db, "DROP TABLE tokens") + end + end + + defp build_token( + {id, external_id, name, workspaces_json, created_at, expires_at, revoked_at, + created_by_type, created_by_external_id} + ) do + %{ + id: id, + external_id: external_id, + name: name, + workspaces: decode_workspaces(workspaces_json), + created_at: created_at, + expires_at: expires_at, + revoked_at: revoked_at, + created_by: build_created_by(created_by_type, created_by_external_id) + } + end + + defp build_created_by(nil, nil), do: nil + defp build_created_by(type, external_id), do: %{type: type, external_id: external_id} + + defp decode_workspaces(nil), do: nil + defp decode_workspaces(json), do: Jason.decode!(json) + + defp encode_workspaces(nil), do: nil + defp encode_workspaces(workspaces), do: Jason.encode!(workspaces) +end diff --git a/server/lib/coflux/orchestration/epoch.ex b/server/lib/coflux/orchestration/epoch.ex index 8677ca10..6426555b 100644 --- a/server/lib/coflux/orchestration/epoch.ex +++ b/server/lib/coflux/orchestration/epoch.ex @@ -1708,77 +1708,27 @@ defmodule Coflux.Orchestration.Epoch do defp ensure_principal(_source_db, _target_db, nil), do: nil + # A principal names its user or token by external id, so it needs nothing + # else copied to exist in the target. defp ensure_principal(source_db, target_db, old_id) do - case query_one!( - source_db, - "SELECT user_external_id, token_id FROM principals WHERE id = ?1", - {old_id} - ) do - {:ok, {user_ext_id, nil}} -> - # User principal — find or create by user_external_id - case query_one( - target_db, - "SELECT id FROM principals WHERE user_external_id = ?1", - {user_ext_id} - ) do - {:ok, {existing_id}} -> - existing_id - - {:ok, nil} -> - {:ok, new_id} = - insert_one(target_db, :principals, %{user_external_id: user_ext_id}) - - new_id - end - - {:ok, {nil, token_id}} -> - # Token principal — ensure the token first, then find or create principal - new_token_id = ensure_token(source_db, target_db, token_id) - - case query_one(target_db, "SELECT id FROM principals WHERE token_id = ?1", {new_token_id}) do - {:ok, {existing_id}} -> - existing_id - - {:ok, nil} -> - {:ok, new_id} = - insert_one(target_db, :principals, %{token_id: new_token_id}) - - new_id - end - end - end - - defp ensure_token(source_db, target_db, old_id) do - {:ok, {ext_id, token_hash, name, workspaces, created_by, created_at, expires_at}} = + {:ok, {user_ext_id, token_ext_id}} = query_one!( source_db, - """ - SELECT external_id, token_hash, name, workspaces, - created_by, created_at, expires_at - FROM tokens - WHERE id = ?1 - """, + "SELECT user_external_id, token_external_id FROM principals WHERE id = ?1", {old_id} ) - case query_one(target_db, "SELECT id FROM tokens WHERE external_id = ?1", {ext_id}) do + {column, ext_id} = + if user_ext_id, + do: {:user_external_id, user_ext_id}, + else: {:token_external_id, token_ext_id} + + case query_one(target_db, "SELECT id FROM principals WHERE #{column} = ?1", {ext_id}) do {:ok, {existing_id}} -> existing_id {:ok, nil} -> - new_created_by = ensure_principal(source_db, target_db, created_by) - - {:ok, new_id} = - insert_one(target_db, :tokens, %{ - external_id: ext_id, - token_hash: token_hash, - name: name, - workspaces: workspaces, - created_by: new_created_by, - created_at: created_at, - expires_at: expires_at - }) - + {:ok, new_id} = insert_one(target_db, :principals, %{column => ext_id}) new_id end end diff --git a/server/lib/coflux/orchestration/principals.ex b/server/lib/coflux/orchestration/principals.ex index 4cf7dfae..35e7dfe5 100644 --- a/server/lib/coflux/orchestration/principals.ex +++ b/server/lib/coflux/orchestration/principals.ex @@ -1,11 +1,11 @@ defmodule Coflux.Orchestration.Principals do @moduledoc """ - Manages principals (users and tokens) in the orchestration database. + Who did something. A principal names a user or a token by external id, + and everything with a `created_by` points at a row here. - A principal represents an authenticated identity - either a user (from JWT) - or a token (from service token auth). - - Users have `user_external_id` set (from JWT sub claim) - - Tokens have `token_id` set, with `external_id` on the tokens table + Tokens themselves live in the admin store (`Coflux.Admin.Tokens`), which + isn't rotated, so a token's principal is made here when the token is + first seen in an epoch rather than when the token is created. """ alias Coflux.Store @@ -14,23 +14,35 @@ defmodule Coflux.Orchestration.Principals do Looks up or creates a user principal by external_id (JWT sub claim). Returns {:ok, principal_id}. """ - def ensure_user(db, external_id) do - case Store.query_one( - db, - "SELECT id FROM principals WHERE user_external_id = ?1", - {external_id} - ) do + def ensure_user(db, external_id), do: ensure(db, :user_external_id, external_id) + + @doc """ + Looks up or creates a token principal by the token's external_id. + Returns {:ok, principal_id}. + """ + def ensure_token(db, external_id), do: ensure(db, :token_external_id, external_id) + + @doc """ + Looks up or creates the principal for an identity as `build/2` describes + it. Nobody (nil) has no principal. + """ + def ensure_identity(_db, nil), do: {:ok, nil} + def ensure_identity(db, %{type: "user", external_id: id}), do: ensure_user(db, id) + def ensure_identity(db, %{type: "token", external_id: id}), do: ensure_token(db, id) + + defp ensure(db, column, external_id) do + case Store.query_one(db, "SELECT id FROM principals WHERE #{column} = ?1", {external_id}) do {:ok, {id}} -> {:ok, id} {:ok, nil} -> - Store.insert_one(db, :principals, %{user_external_id: external_id}) + Store.insert_one(db, :principals, %{column => external_id}) end end @doc """ Who did something, from the external ids of the user and token a - `principals` join yields. At most one is set; neither means nobody - a + `principals` row holds. At most one is set; neither means nobody - a run the server started on its own. """ def build(nil, nil), do: nil @@ -44,14 +56,11 @@ defmodule Coflux.Orchestration.Principals do def get_principal(_db, nil), do: {:ok, nil} def get_principal(db, principal_id) do - query = """ - SELECT p.user_external_id, t.external_id - FROM principals p - LEFT JOIN tokens t ON p.token_id = t.id - WHERE p.id = ?1 - """ - - case Store.query_one(db, query, {principal_id}) do + case Store.query_one( + db, + "SELECT user_external_id, token_external_id FROM principals WHERE id = ?1", + {principal_id} + ) do {:ok, {user_external_id, nil}} -> {:ok, {"user", user_external_id}} @@ -62,197 +71,4 @@ defmodule Coflux.Orchestration.Principals do {:ok, nil} end end - - @doc """ - Validates a token by hash and returns the principal_id and workspaces if valid. - Returns {:ok, %{principal_id: id, workspaces: list | nil}} or {:error, :not_found}. - Workspaces is nil for all-access, or a list of patterns. - """ - def check_token(db, token_hash) do - now = System.system_time(:second) - - query = """ - SELECT p.id, t.workspaces - FROM tokens t - JOIN principals p ON p.token_id = t.id - WHERE t.token_hash = ?1 - AND t.revoked_at IS NULL - AND (t.expires_at IS NULL OR t.expires_at > ?2) - """ - - case Store.query_one(db, query, {token_hash, now}) do - {:ok, {principal_id, workspaces_json}} -> - workspaces = decode_workspaces(workspaces_json) - {:ok, %{principal_id: principal_id, workspaces: workspaces}} - - {:ok, nil} -> - {:error, :not_found} - end - end - - defp decode_workspaces(nil), do: nil - defp decode_workspaces(json), do: Jason.decode!(json) - - @doc """ - Creates a new token and its associated principal. - Returns {:ok, %{token: token, token_id: token_id, principal_id: principal_id, external_id: external_id}}. - - Options: - - workspaces: list of workspace patterns, or nil for all workspaces - """ - def create_token(db, project_id, name, created_by, opts \\ []) do - now = System.system_time(:second) - workspaces = Keyword.get(opts, :workspaces) - - # Generate random bytes (16 bytes = 32 hex chars) - random = :crypto.strong_rand_bytes(16) - random_hex = Base.encode16(random, case: :lower) - - # Compute signature (first 4 bytes of HMAC) - signature_hex = Coflux.Auth.compute_token_signature(random_hex, project_id) - - # Full token: cflx_ - token = "cflx_" <> random_hex <> signature_hex - - # Store hash of random part for lookup - random_hash = :crypto.hash(:sha256, random_hex) |> Base.encode16(case: :lower) - - # Generate external_id for the token - {:ok, external_id} = Store.generate_external_id(db, "tokens", 12) - - # Insert token with external_id - {:ok, token_id} = - Store.insert_one(db, :tokens, %{ - external_id: external_id, - token_hash: random_hash, - name: name, - workspaces: encode_workspaces(workspaces), - created_by: created_by, - created_at: now - }) - - # Create principal for this token - {:ok, principal_id} = - Store.insert_one(db, :principals, %{ - token_id: token_id - }) - - {:ok, created_by} = resolve_created_by(db, created_by) - - {:ok, - %{ - id: token_id, - token: token, - token_id: token_id, - principal_id: principal_id, - external_id: external_id, - name: name, - workspaces: workspaces, - created_at: now, - expires_at: nil, - revoked_at: nil, - created_by: created_by - }} - end - - defp resolve_created_by(_db, nil), do: {:ok, nil} - - defp resolve_created_by(db, principal_id) do - case get_principal(db, principal_id) do - {:ok, {type, ext_id}} -> {:ok, %{type: type, external_id: ext_id}} - {:ok, nil} -> {:ok, nil} - end - end - - defp encode_workspaces(nil), do: nil - defp encode_workspaces(workspaces), do: Jason.encode!(workspaces) - - @doc """ - Lists all tokens with their metadata. - """ - def list_tokens(db) do - query = """ - SELECT t.id, t.external_id, t.name, t.workspaces, t.created_at, t.expires_at, t.revoked_at, - t.created_by, p_creator.user_external_id, t_creator.external_id - FROM tokens t - LEFT JOIN principals p_creator ON t.created_by = p_creator.id - LEFT JOIN tokens t_creator ON p_creator.token_id = t_creator.id - ORDER BY t.created_at DESC - """ - - {:ok, rows} = Store.query(db, query, {}) - - {:ok, Enum.map(rows, &build_token/1)} - end - - defp build_token( - {id, external_id, name, workspaces_json, created_at, expires_at, revoked_at, - created_by_principal_id, creator_user_ext_id, creator_token_ext_id} - ) do - created_by = - case {creator_user_ext_id, creator_token_ext_id} do - {nil, nil} -> nil - {user_ext_id, nil} -> %{type: "user", external_id: user_ext_id} - {nil, token_ext_id} -> %{type: "token", external_id: token_ext_id} - end - - %{ - id: id, - external_id: external_id, - name: name, - workspaces: decode_workspaces(workspaces_json), - created_at: created_at, - expires_at: expires_at, - revoked_at: revoked_at, - created_by: created_by, - created_by_principal_id: created_by_principal_id - } - end - - @doc """ - Gets a token by its external_id. - Returns {:ok, token} or {:ok, nil} if not found. - """ - def get_token_by_external_id(db, external_id) do - query = """ - SELECT t.id, t.external_id, t.name, t.created_at, t.expires_at, t.revoked_at, t.created_by - FROM tokens t - WHERE t.external_id = ?1 - """ - - case Store.query_one(db, query, {external_id}) do - {:ok, {id, ext_id, name, created_at, expires_at, revoked_at, created_by_principal_id}} -> - {:ok, - %{ - id: id, - external_id: ext_id, - name: name, - created_at: created_at, - expires_at: expires_at, - revoked_at: revoked_at, - created_by_principal_id: created_by_principal_id - }} - - {:ok, nil} -> - {:ok, nil} - end - end - - @doc """ - Revokes a token by ID. - """ - def revoke_token(db, token_id) do - now = System.system_time(:second) - - case Store.query_one(db, "SELECT id, external_id FROM tokens WHERE id = ?1", {token_id}) do - {:ok, {^token_id, external_id}} -> - {:ok, _} = - Store.query(db, "UPDATE tokens SET revoked_at = ?1 WHERE id = ?2", {now, token_id}) - - {:ok, external_id} - - {:ok, nil} -> - {:error, :not_found} - end - end end diff --git a/server/lib/coflux/orchestration/results.ex b/server/lib/coflux/orchestration/results.ex index c6290607..52ce68ab 100644 --- a/server/lib/coflux/orchestration/results.ex +++ b/server/lib/coflux/orchestration/results.ex @@ -378,10 +378,9 @@ defmodule Coflux.Orchestration.Results do db, """ SELECT c.kind, c.successor_id, c.successor_ref_id, c.created_at, - p.user_external_id, t.external_id + p.user_external_id, p.token_external_id FROM completions AS c LEFT JOIN principals AS p ON c.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE c.execution_id = ?1 """, {execution_id} @@ -483,10 +482,9 @@ defmodule Coflux.Orchestration.Results do db, """ SELECT c.kind, c.successor_id, c.successor_ref_id, c.created_at, - p.user_external_id, t.external_id + p.user_external_id, p.token_external_id FROM completions AS c LEFT JOIN principals AS p ON c.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE c.execution_id = ?1 """, {execution_id} diff --git a/server/lib/coflux/orchestration/runs.ex b/server/lib/coflux/orchestration/runs.ex index 3b3f0e8f..f7bc9eb2 100644 --- a/server/lib/coflux/orchestration/runs.ex +++ b/server/lib/coflux/orchestration/runs.ex @@ -813,7 +813,7 @@ defmodule Coflux.Orchestration.Runs do s.requires_tag_set_id AS step_requires_tag_set_id, r.requires_tag_set_id AS run_requires_tag_set_id, p.user_external_id AS created_by_user_external_id, - t.external_id AS created_by_token_external_id + p.token_external_id AS created_by_token_external_id """ @execution_joins """ @@ -825,7 +825,6 @@ defmodule Coflux.Orchestration.Runs do LEFT JOIN assignments AS a ON a.execution_id = e.id LEFT JOIN completions AS c ON c.execution_id = e.id LEFT JOIN principals AS p ON e.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id """ @doc """ @@ -996,7 +995,7 @@ defmodule Coflux.Orchestration.Runs do """ SELECT DISTINCT r.external_id, r.created_at, p.user_external_id AS created_by_user_external_id, - t.external_id AS created_by_token_external_id, + p.token_external_id AS created_by_token_external_id, ( SELECT e2.id FROM executions AS e2 WHERE e2.step_id = s.id @@ -1007,7 +1006,6 @@ defmodule Coflux.Orchestration.Runs do INNER JOIN steps AS s ON s.run_id = r.id INNER JOIN executions AS e ON e.step_id == s.id LEFT JOIN principals AS p ON r.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE s.module = ?1 AND s.target = ?2 AND s.type = ?3 AND s.parent_id IS NULL AND e.workspace_id = ?4 ORDER BY r.created_at DESC LIMIT ?5 @@ -1022,10 +1020,9 @@ defmodule Coflux.Orchestration.Runs do """ SELECT r.id, r.external_id, r.parent_ref_id, r.idempotency_key, r.requires_tag_set_id, r.memo, r.created_at, p.user_external_id AS created_by_user_external_id, - t.external_id AS created_by_token_external_id + p.token_external_id AS created_by_token_external_id FROM runs AS r LEFT JOIN principals AS p ON r.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE r.id = ?1 """, {id}, @@ -1039,10 +1036,9 @@ defmodule Coflux.Orchestration.Runs do """ SELECT r.id, r.external_id, r.parent_ref_id, r.idempotency_key, r.requires_tag_set_id, r.memo, r.created_at, p.user_external_id AS created_by_user_external_id, - t.external_id AS created_by_token_external_id + p.token_external_id AS created_by_token_external_id FROM runs AS r LEFT JOIN principals AS p ON r.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE r.external_id = ?1 """, {external_id}, @@ -1200,12 +1196,11 @@ defmodule Coflux.Orchestration.Runs do """ SELECT e.id, e.step_id, e.attempt, e.workspace_id, e.execute_after, e.created_at, a.created_at, p.user_external_id AS created_by_user_external_id, - t.external_id AS created_by_token_external_id + p.token_external_id AS created_by_token_external_id FROM steps AS s INNER JOIN executions AS e ON e.step_id = s.id LEFT JOIN assignments AS a ON a.execution_id = e.id LEFT JOIN principals AS p ON e.created_by = p.id - LEFT JOIN tokens AS t ON p.token_id = t.id WHERE s.run_id = ?1 """, {run_id} diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index 46b01201..d6c2cea0 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -98,9 +98,14 @@ defmodule Coflux.Orchestration.Server do {:ok, epochs} -> db = Epochs.active_db(epochs) + # The admin store isn't an epoch: it holds what outlives them. + {:ok, admin_db} = Coflux.Store.open(project_id, "admin") + :ok = Coflux.Admin.Tokens.import_legacy(db, admin_db) + state = %State{ project_id: project_id, db: db, + admin_db: admin_db, epochs: epochs, epoch_index: epoch_index, index_queue: unindexed_epoch_ids @@ -253,17 +258,30 @@ defmodule Coflux.Orchestration.Server do # Token management defp dispatch_call({:check_token, token_hash}, state) do - case Principals.check_token(state.db, token_hash) do - {:ok, %{principal_id: principal_id, workspaces: workspaces}} -> - {:reply, {:ok, %{workspaces: workspaces, principal_id: principal_id}}, state} - + with {:ok, %{external_id: external_id, workspaces: workspaces}} <- + Coflux.Admin.Tokens.check_token(state.admin_db, token_hash), + {:ok, principal_id} <- Principals.ensure_token(state.db, external_id) do + {:reply, {:ok, %{workspaces: workspaces, principal_id: principal_id}}, state} + else {:error, :not_found} -> {:reply, {:error, :not_found}, state} end end defp dispatch_call({:create_token, name, principal_id, opts}, state) do - {:ok, result} = Principals.create_token(state.db, state.project_id, name, principal_id, opts) + created_by = + case Principals.get_principal(state.db, principal_id) do + {:ok, {type, external_id}} -> %{type: type, external_id: external_id} + {:ok, nil} -> nil + end + + {:ok, result} = + Coflux.Admin.Tokens.create_token(state.admin_db, state.project_id, name, created_by, opts) + + # The token can act straight away, so give it its principal now rather + # than on first use. + {:ok, token_principal_id} = Principals.ensure_token(state.db, result.external_id) + result = Map.put(result, :principal_id, token_principal_id) state = state @@ -281,12 +299,12 @@ defmodule Coflux.Orchestration.Server do end defp dispatch_call(:list_tokens, state) do - {:ok, tokens} = Principals.list_tokens(state.db) + {:ok, tokens} = Coflux.Admin.Tokens.list_tokens(state.admin_db) {:reply, {:ok, tokens}, state} end defp dispatch_call({:revoke_token, token_id}, state) do - case Principals.revoke_token(state.db, token_id) do + case Coflux.Admin.Tokens.revoke_token(state.admin_db, token_id) do {:ok, external_id} -> state = Effects.emit(state, %TokenRevoked{token: external_id}) {:reply, {:ok, external_id}, state} @@ -297,8 +315,18 @@ defmodule Coflux.Orchestration.Server do end defp dispatch_call({:get_token, external_id}, state) do - result = Principals.get_token_by_external_id(state.db, external_id) - {:reply, result, state} + case Coflux.Admin.Tokens.get_token_by_external_id(state.admin_db, external_id) do + {:ok, nil} -> + {:reply, {:ok, nil}, state} + + {:ok, token} -> + # Whether the caller may revoke it is decided by principal id, and + # a principal is local to this epoch, so the creator's identity is + # given one here if it hasn't one yet. + {:ok, created_by_principal_id} = Principals.ensure_identity(state.db, token.created_by) + token = Map.put(token, :created_by_principal_id, created_by_principal_id) + {:reply, {:ok, token}, state} + end end # Workspace management @@ -2891,6 +2919,10 @@ defmodule Coflux.Orchestration.Server do if state.epochs do Epochs.close(state.epochs) end + + if state.admin_db do + Coflux.Store.close(state.admin_db) + end end # Private helper functions diff --git a/server/lib/coflux/orchestration/server/archives.ex b/server/lib/coflux/orchestration/server/archives.ex index ea167933..ffb49c41 100644 --- a/server/lib/coflux/orchestration/server/archives.ex +++ b/server/lib/coflux/orchestration/server/archives.ex @@ -388,9 +388,7 @@ defmodule Coflux.Orchestration.Server.Archives do |> Enum.reject(&MapSet.member?(unindexed_ids, &1)) Enum.reduce_while(candidate_epoch_ids, :not_found, fn epoch_id, :not_found -> - path = Epochs.archive_path(state.epochs, epoch_id) - - case Exqlite.Sqlite3.open(path) do + case Epochs.open_archive(state.epochs, epoch_id) do {:ok, archive_db} -> try do case query_epoch(state, epoch_id, archive_db, query_fn) do diff --git a/server/lib/coflux/orchestration/server/rotation.ex b/server/lib/coflux/orchestration/server/rotation.ex index fdd4bbcc..6c6f7135 100644 --- a/server/lib/coflux/orchestration/server/rotation.ex +++ b/server/lib/coflux/orchestration/server/rotation.ex @@ -100,11 +100,11 @@ defmodule Coflux.Orchestration.Server.Rotation do end def maybe_start_index_build(%{index_task: nil, index_queue: [epoch_id | _]} = state) do - path = Epochs.archive_path(state.epochs, epoch_id) + epochs = state.epochs task = Task.Supervisor.async_nolink(Coflux.LauncherSupervisor, fn -> - {:ok, db} = Exqlite.Sqlite3.open(path) + {:ok, db} = Epochs.open_archive(epochs, epoch_id) try do build_blooms_for_epoch(db) diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index bc06ee28..f8f76bfa 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -209,7 +209,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do # Revoked tokens are absent: created-then-revoked folds to absence. def load(state, :tokens, _opts) do - {:ok, tokens} = Principals.list_tokens(state.db) + {:ok, tokens} = Coflux.Admin.Tokens.list_tokens(state.admin_db) events = tokens @@ -411,9 +411,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do workspace_external_id, limit ) do - path = Epochs.archive_path(state.epochs, epoch_id) - - case Exqlite.Sqlite3.open(path) do + case Epochs.open_archive(state.epochs, epoch_id) do {:ok, db} -> try do query_archive_target_runs(db, module, target, workspace_external_id, limit) diff --git a/server/lib/coflux/orchestration/server/state.ex b/server/lib/coflux/orchestration/server/state.ex index 34f1a29e..1c9f2f08 100644 --- a/server/lib/coflux/orchestration/server/state.ex +++ b/server/lib/coflux/orchestration/server/state.ex @@ -37,6 +37,10 @@ defmodule Coflux.Orchestration.Server.State do # by `Rotation`, so nothing may hold onto it across a call. db: nil, + # The admin store: tokens, and whatever else outlives an + # epoch. Not rotated, so unlike `db` it never changes hands. + admin_db: nil, + # `Coflux.Store.Epochs`: the active database plus the ids of # the archived epochs behind it. `Archives` searches backwards # through those for rows the active epoch no longer holds. diff --git a/server/lib/coflux/store/epochs.ex b/server/lib/coflux/store/epochs.ex index 955fcfc1..e60111ef 100644 --- a/server/lib/coflux/store/epochs.ex +++ b/server/lib/coflux/store/epochs.ex @@ -55,6 +55,7 @@ defmodule Coflux.Store.Epochs do if File.exists?(path) do {:ok, db} = Sqlite3.open(path) + :ok = Migrations.run(db, name) [{epoch_id, db}] else [] @@ -71,6 +72,20 @@ defmodule Coflux.Store.Epochs do }} end + @doc """ + Opens an archived epoch's database, bringing its schema up to date first. + + Archives are read with the same queries as the active database, so they + have to have the same shape: a migration that adds a column is as + necessary here as there. The caller closes the handle. + """ + def open_archive(%__MODULE__{} = epochs, epoch_id) do + with {:ok, db} <- Sqlite3.open(archive_path(epochs, epoch_id)) do + :ok = Migrations.run(db, epochs.name) + {:ok, db} + end + end + @doc """ Generate the next epoch ID based on today's date and existing archived IDs. """ diff --git a/server/lib/coflux/store/migrations.ex b/server/lib/coflux/store/migrations.ex index 05231906..c41ac31d 100644 --- a/server/lib/coflux/store/migrations.ex +++ b/server/lib/coflux/store/migrations.ex @@ -3,7 +3,13 @@ defmodule Coflux.Store.Migrations do @otp_app Mix.Project.config()[:app] - def run(db, name) do + @doc """ + Brings a database up to date with the migrations for `name`. + + Options: + - `:up_to` - stop after this version (for tests that need an older shape) + """ + def run(db, name, opts \\ []) do migrations_dir = @otp_app |> Application.app_dir("priv/migrations") @@ -11,8 +17,15 @@ defmodule Coflux.Store.Migrations do setup_migrations_table(db) - migrations_dir - |> get_available_versions() + available = get_available_versions(migrations_dir) + + available = + case Keyword.get(opts, :up_to) do + nil -> available + up_to -> MapSet.filter(available, &(&1 <= up_to)) + end + + available |> MapSet.difference(get_migrated_versions(db)) |> Enum.sort() |> Enum.each(&run_migration(db, migrations_dir, &1)) @@ -53,10 +66,18 @@ defmodule Coflux.Store.Migrations do |> Path.join("#{version}.sql") |> File.read!() - :ok = Sqlite3.execute(db, "BEGIN") - :ok = Sqlite3.execute(db, sql) - :ok = insert_schema_migration(db, version) - :ok = Sqlite3.execute(db, "COMMIT") + # Take the write lock before looking again: two handles on one file - + # an archive opened for a query while its index is being built, say - + # can both find the same migration pending, and only one may apply it. + :ok = Sqlite3.execute(db, "BEGIN IMMEDIATE") + + if MapSet.member?(get_migrated_versions(db), version) do + :ok = Sqlite3.execute(db, "COMMIT") + else + :ok = Sqlite3.execute(db, sql) + :ok = insert_schema_migration(db, version) + :ok = Sqlite3.execute(db, "COMMIT") + end end defp insert_schema_migration(db, version) do diff --git a/server/priv/migrations/admin/1.sql b/server/priv/migrations/admin/1.sql new file mode 100644 index 00000000..c0ae3202 --- /dev/null +++ b/server/priv/migrations/admin/1.sql @@ -0,0 +1,19 @@ +-- The admin store: identity and configuration that outlive any epoch. +-- Unlike the orchestration store it is never rotated, so what is here is +-- here once, and deleting something deletes it. + +-- Service tokens. A token's principal lives in the orchestration store +-- (named by `external_id`), where everything it does is attributed. +CREATE TABLE tokens ( + id INTEGER PRIMARY KEY, + external_id TEXT NOT NULL UNIQUE, + token_hash TEXT NOT NULL UNIQUE, + name TEXT, + workspaces TEXT, -- JSON array of workspace patterns, NULL for all workspaces + created_by_type TEXT, -- 'user' or 'token' + created_by_external_id TEXT, + created_at INTEGER NOT NULL, + expires_at INTEGER, + revoked_at INTEGER, + CHECK ((created_by_type IS NULL) = (created_by_external_id IS NULL)) +) STRICT; diff --git a/server/priv/migrations/orchestration/6.sql b/server/priv/migrations/orchestration/6.sql index 8d4a90c4..b25c5ef2 100644 --- a/server/priv/migrations/orchestration/6.sql +++ b/server/priv/migrations/orchestration/6.sql @@ -138,3 +138,25 @@ ALTER TABLE executions ADD COLUMN catalog_sequence INTEGER; -- How long a pool keeps an idle worker before stopping it, in seconds. -- NULL leaves it to the scheduler's default. ALTER TABLE pool_definitions ADD COLUMN idle_timeout INTEGER; + +-- Tokens have moved to the admin store, which isn't rotated, so a +-- principal names its token by external id rather than by a row in this +-- database - the way it already names a user. +-- +-- The `tokens` table stays for now: the server copies it across to the +-- admin store when it starts, then drops it (`Coflux.Admin.Tokens`). +CREATE TABLE principals_new ( + id INTEGER PRIMARY KEY, + user_external_id TEXT UNIQUE, + token_external_id TEXT UNIQUE, + CHECK ((user_external_id IS NOT NULL AND token_external_id IS NULL) OR (user_external_id IS NULL AND token_external_id IS NOT NULL)) +) STRICT; + +INSERT INTO principals_new (id, user_external_id, token_external_id) +SELECT p.id, p.user_external_id, t.external_id +FROM principals AS p +LEFT JOIN tokens AS t ON t.id = p.token_id +WHERE p.user_external_id IS NOT NULL OR t.external_id IS NOT NULL; + +DROP TABLE principals; +ALTER TABLE principals_new RENAME TO principals; diff --git a/server/test/coflux/admin_tokens_test.exs b/server/test/coflux/admin_tokens_test.exs new file mode 100644 index 00000000..136eb49a --- /dev/null +++ b/server/test/coflux/admin_tokens_test.exs @@ -0,0 +1,115 @@ +defmodule Coflux.AdminTokensTest do + use ExUnit.Case, async: true + + alias Coflux.Admin.Tokens + alias Coflux.Orchestration.Principals + alias Coflux.Store + alias Coflux.Store.Migrations + alias Exqlite.Sqlite3 + + # Creating a token signs it with the server secret. + setup_all do + previous = :persistent_term.get(:coflux_secret, nil) + :persistent_term.put(:coflux_secret, "test-secret") + on_exit(fn -> :persistent_term.put(:coflux_secret, previous) end) + :ok + end + + # An orchestration database as it was before tokens moved: at version 5, + # with a token, its principal, and a user principal that created it. + defp legacy_orchestration_db do + {:ok, db} = Sqlite3.open(":memory:") + :ok = Migrations.run(db, "orchestration", up_to: 5) + + {:ok, user_id} = Store.insert_one(db, :principals, %{user_external_id: "user-1"}) + + {:ok, token_id} = + Store.insert_one(db, :tokens, %{ + external_id: "tok1", + token_hash: "hash1", + name: "CI", + workspaces: ~s(["staging"]), + created_by: user_id, + created_at: 1000 + }) + + {:ok, _} = Store.insert_one(db, :principals, %{token_id: token_id}) + db + end + + defp admin_db do + {:ok, db} = Sqlite3.open(":memory:") + :ok = Migrations.run(db, "admin") + db + end + + test "migrating rebuilds principals to name tokens by external id" do + db = legacy_orchestration_db() + :ok = Migrations.run(db, "orchestration") + + {:ok, rows} = + Store.query( + db, + "SELECT user_external_id, token_external_id FROM principals ORDER BY id", + {} + ) + + assert rows == [{"user-1", nil}, {nil, "tok1"}] + assert {:ok, {"token", "tok1"}} = Principals.get_principal(db, 2) + end + + test "importing moves tokens to the admin store and drops the table" do + db = legacy_orchestration_db() + :ok = Migrations.run(db, "orchestration") + admin = admin_db() + + :ok = Tokens.import_legacy(db, admin) + + assert {:ok, [token]} = Tokens.list_tokens(admin) + assert token.external_id == "tok1" + assert token.name == "CI" + assert token.workspaces == ["staging"] + assert token.created_by == %{type: "user", external_id: "user-1"} + assert token.revoked_at == nil + + assert {:ok, nil} = + Store.query_one( + db, + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'tokens'", + {} + ) + + # Nothing left to import, and nothing duplicated. + :ok = Tokens.import_legacy(db, admin) + assert {:ok, [_]} = Tokens.list_tokens(admin) + end + + test "a token is checked in the admin store and attributed in the orchestration store" do + {:ok, db} = Sqlite3.open(":memory:") + :ok = Migrations.run(db, "orchestration") + admin = admin_db() + + {:ok, created} = + Tokens.create_token(admin, "proj", "deploy", %{type: "user", external_id: "user-1"}, + workspaces: ["prod"] + ) + + "cflx_" <> rest = created.token + random_hex = binary_part(rest, 0, 32) + hash = :crypto.hash(:sha256, random_hex) |> Base.encode16(case: :lower) + + assert {:ok, %{external_id: external_id, workspaces: ["prod"]}} = + Tokens.check_token(admin, hash) + + assert external_id == created.external_id + + # The principal is made on first sight and found afterwards. + {:ok, principal_id} = Principals.ensure_token(db, external_id) + assert {:ok, ^principal_id} = Principals.ensure_token(db, external_id) + + assert {:ok, {"token", ^external_id}} = Principals.get_principal(db, principal_id) + + {:ok, ^external_id} = Tokens.revoke_token(admin, created.id) + assert {:error, :not_found} = Tokens.check_token(admin, hash) + end +end diff --git a/server/test/coflux/migrations_test.exs b/server/test/coflux/migrations_test.exs index 800fa5a4..c559342b 100644 --- a/server/test/coflux/migrations_test.exs +++ b/server/test/coflux/migrations_test.exs @@ -12,5 +12,13 @@ defmodule Coflux.MigrationsTest do :ok = Sqlite3.close(db) end + + test "evaluates admin migrations" do + {:ok, db} = Sqlite3.open(":memory:") + + assert :ok = Migrations.run(db, "admin") + + :ok = Sqlite3.close(db) + end end end diff --git a/tests/test_tokens.py b/tests/test_tokens.py new file mode 100644 index 00000000..dd09be32 --- /dev/null +++ b/tests/test_tokens.py @@ -0,0 +1,109 @@ +"""Service tokens. + +Tokens live in the admin store, which isn't rotated, so a token outlives +epoch rotations and server restarts, and is gone for good once revoked. +""" + +import json +import tempfile +import urllib.error +import urllib.request +import uuid + +import pytest +from support.helpers import api_post +from support.server import ManagedServer + + +@pytest.fixture(scope="module") +def token_server(): + """A server with authentication on and a secret to sign tokens with.""" + data_dir = tempfile.mkdtemp(prefix="coflux-test-tokens-") + srv = ManagedServer( + data_dir, + extra_env={ + "COFLUX_REQUIRE_AUTH": "true", + "COFLUX_SECRET": "test-secret-for-service-tokens", + }, + ) + srv.start(timeout=30) + yield srv + srv.stop() + + +def _discover(port, project_id, token): + """What a token is granted: (status, workspaces or None).""" + req = urllib.request.Request( + f"http://{project_id}.localhost:{port}/api/discover", + headers={"Authorization": f"Bearer {token}"}, + ) + try: + with urllib.request.urlopen(req, timeout=5) as resp: + return resp.status, json.loads(resp.read())["access"]["workspaces"] + except urllib.error.HTTPError as e: + return e.code, None + + +def _create(port, project_id, token=None, **body): + return api_post(port, project_id, "create_token", token=token, body=body) + + +def _revoke(port, project_id, external_id, token=None): + """Returns the HTTP status.""" + try: + api_post( + port, + project_id, + "revoke_token", + token=token, + body={"externalId": external_id}, + ) + return 204 + except urllib.error.HTTPError as e: + return e.code + + +class TestServiceTokens: + def test_token_grants_its_workspaces_until_revoked(self, token_server): + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + + created = _create(port, project_id, name="ci", workspaces=["staging"]) + assert _discover(port, project_id, created["token"]) == (200, ["staging"]) + + assert _revoke(port, project_id, created["externalId"]) == 204 + assert _discover(port, project_id, created["token"]) == (401, None) + + def test_token_outlives_rotation_and_restart(self, token_server): + """The admin store isn't an epoch: rotating one, or restarting the + server, changes nothing about which tokens exist.""" + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + + created = _create(port, project_id, name="deploy") + api_post(port, project_id, "rotate_epoch") + assert _discover(port, project_id, created["token"]) == (200, ["*"]) + + token_server.restart(timeout=30) + assert _discover(port, project_id, created["token"]) == (200, ["*"]) + + def test_only_the_creator_or_full_access_can_revoke(self, token_server): + """Who created a token is kept with it, and resolved to a principal + in whichever epoch asks - here, after a rotation.""" + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + + parent = _create(port, project_id, name="parent", workspaces=["staging"]) + other = _create(port, project_id, name="other", workspaces=["staging"]) + child = _create(port, project_id, token=parent["token"], name="child") + + api_post(port, project_id, "rotate_epoch") + + assert ( + _revoke(port, project_id, child["externalId"], token=other["token"]) == 403 + ) + assert _discover(port, project_id, child["token"]) == (200, ["staging"]) + assert ( + _revoke(port, project_id, child["externalId"], token=parent["token"]) == 204 + ) + assert _discover(port, project_id, child["token"]) == (401, None) From f20f257b894776dac027150c02ad4502bee85136 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sat, 19 Sep 2026 21:50:53 +0100 Subject: [PATCH 06/21] Support managing secrets --- cli/CHANGELOG.md | 1 + cli/cmd/coflux/main.go | 3 +- cli/cmd/coflux/pools.go | 96 ++--- cli/cmd/coflux/secrets.go | 236 +++++++++++ cli/internal/api/client.go | 44 ++- docs/docs/cli_reference.md | 10 +- docs/docs/pools.md | 83 ++-- docs/docs/server_config.md | 6 +- server/lib/coflux/admin/secrets.ex | 368 ++++++++++++++++++ server/lib/coflux/application.ex | 1 + server/lib/coflux/events.ex | 13 + server/lib/coflux/handlers/api.ex | 206 ++++++---- server/lib/coflux/launchers/docker.ex | 4 +- server/lib/coflux/launchers/ecs.ex | 45 +-- server/lib/coflux/launchers/kubernetes.ex | 18 +- server/lib/coflux/launchers/process.ex | 4 +- server/lib/coflux/orchestration.ex | 8 + server/lib/coflux/orchestration/server.ex | 84 +++- .../coflux/orchestration/server/routing.ex | 4 + .../coflux/orchestration/server/scheduler.ex | 325 ++++++++++------ .../coflux/orchestration/server/snapshots.ex | 17 + server/lib/coflux/orchestration/workspaces.ex | 29 +- server/lib/coflux/topics/pool.ex | 16 +- server/lib/coflux/topics/secrets.ex | 94 +++++ server/priv/migrations/admin/2.sql | 24 ++ server/test/coflux/admin_secrets_test.exs | 134 +++++++ tests/support/cli.py | 40 +- tests/support/server.py | 4 + tests/test_pools.py | 205 +++++++--- tests/test_secrets.py | 60 +++ tests/test_tokens.py | 24 ++ 31 files changed, 1799 insertions(+), 407 deletions(-) create mode 100644 cli/cmd/coflux/secrets.go create mode 100644 server/lib/coflux/admin/secrets.ex create mode 100644 server/lib/coflux/topics/secrets.ex create mode 100644 server/priv/migrations/admin/2.sql create mode 100644 server/test/coflux/admin_secrets_test.exs create mode 100644 tests/test_secrets.py diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index ad6616ae..004457bd 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -4,6 +4,7 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`. - Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it. +- Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. ## 0.12.0 diff --git a/cli/cmd/coflux/main.go b/cli/cmd/coflux/main.go index e19adc8f..a30736f4 100644 --- a/cli/cmd/coflux/main.go +++ b/cli/cmd/coflux/main.go @@ -105,7 +105,8 @@ func init() { queueCmd.GroupID = "management" inputsCmd.GroupID = "management" catalogCmd.GroupID = "management" - rootCmd.AddCommand(workspacesCmd, manifestsCmd, poolsCmd, tokensCmd, assetsCmd, blobsCmd, logsCmd, sessionsCmd, queueCmd, inputsCmd, catalogCmd) + secretsCmd.GroupID = "management" + rootCmd.AddCommand(workspacesCmd, manifestsCmd, poolsCmd, tokensCmd, secretsCmd, assetsCmd, blobsCmd, logsCmd, sessionsCmd, queueCmd, inputsCmd, catalogCmd) } func initConfig(cmd *cobra.Command, args []string) error { diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index e5218e46..0ba79258 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -191,6 +191,9 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if policy := getString(launcher, "imagePullPolicy"); policy != "" { fmt.Printf("Image pull policy: %s\n", policy) } + if secret := getString(launcher, "tokenSecret"); secret != "" { + fmt.Printf("Token secret: %s\n", secret) + } if cluster := getString(launcher, "cluster"); cluster != "" { fmt.Printf("Cluster: %s\n", cluster) } @@ -220,8 +223,8 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if version := getString(launcher, "platformVersion"); version != "" { fmt.Printf("Platform version: %s\n", version) } - if keyID := getString(launcher, "accessKeyId"); keyID != "" { - fmt.Printf("Access key ID: %s\n", keyID) + if secret := getString(launcher, "credentialsSecret"); secret != "" { + fmt.Printf("Credentials secret: %s\n", secret) } if endpoint := getString(launcher, "endpoint"); endpoint != "" { fmt.Printf("Endpoint: %s\n", endpoint) @@ -239,6 +242,12 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { fmt.Printf(" %s=%s\n", k, v) } } + if secrets, ok := launcher["envSecrets"].(map[string]any); ok && len(secrets) > 0 { + fmt.Printf("Secret environment:\n") + for k, v := range secrets { + fmt.Printf(" %s=<%s>\n", k, v) + } + } } return nil @@ -628,7 +637,7 @@ var poolTopLevelFields = map[string]bool{ var launcherFields = map[string]bool{ "image": true, "dockerHost": true, "networkMode": true, "directory": true, "namespace": true, "serviceAccount": true, "apiServer": true, - "token": true, "caCert": true, "insecure": true, + "tokenSecret": true, "caCert": true, "insecure": true, "imagePullPolicy": true, "nodeSelector": true, "tolerations": true, "imagePullSecrets": true, "hostAliases": true, "resources": true, "labels": true, "annotations": true, "activeDeadlineSeconds": true, @@ -636,16 +645,16 @@ var launcherFields = map[string]bool{ "cluster": true, "taskDefinition": true, "region": true, "containerName": true, "launchType": true, "capacityProvider": true, "subnets": true, "securityGroups": true, "assignPublicIp": true, - "platformVersion": true, "accessKeyId": true, "secretAccessKey": true, - "sessionToken": true, "endpoint": true, + "platformVersion": true, "credentialsSecret": true, "endpoint": true, "serverHost": true, "serverSecure": true, "adapter": true, - "concurrency": true, "env": true, + "concurrency": true, "env": true, "envSecrets": true, } // mapSubkeyFields lists launcher fields that support dotted sub-key access // (e.g. labels.team=ml, annotations.prometheus.io/scrape=true). var mapSubkeyFields = map[string]bool{ "env": true, + "envSecrets": true, "labels": true, "annotations": true, "nodeSelector": true, @@ -989,52 +998,24 @@ func runPoolsEnable(cmd *cobra.Command, args []string) error { return nil } -// redactedSecret is what the server substitutes for a secret the caller -// didn't ask for. It is deliberately not a usable value, so a config -// exported without secrets can't be imported back over the real ones. -const redactedSecret = "" - -// redactedPools names the pools whose config still carries a redacted -// secret, so both export and import can say which ones and why. -func redactedPools(pools map[string]map[string]any) []string { - var names []string - for name, pool := range pools { - launcher, ok := pool["launcher"].(map[string]any) - if !ok { - continue - } - for _, value := range launcher { - if s, ok := value.(string); ok && s == redactedSecret { - names = append(names, name) - break - } - } - } - sort.Strings(names) - return names -} - // pools export var poolsExportOutput string var poolsExportOnly []string -var poolsExportIncludeSecrets bool var poolsExportCmd = &cobra.Command{ Use: "export", Short: "Export pool configuration", Long: `Export all pool configurations for the workspace as TOML. Writes to stdout by default. -Launcher secrets, such as a Kubernetes token or an AWS secret key, are redacted unless --include-secrets -is given. A redacted file is refused by 'pools import', so it cannot silently clear -the secrets it omits.`, +Pools refer to secrets by name, so an export never contains a secret's value. +Importing it elsewhere needs the same secrets to exist there.`, RunE: runPoolsExport, } func init() { poolsExportCmd.Flags().StringVarP(&poolsExportOutput, "output", "o", "", "Output file (default: stdout)") poolsExportCmd.Flags().StringSliceVar(&poolsExportOnly, "only", nil, "Export only named pools") - poolsExportCmd.Flags().BoolVar(&poolsExportIncludeSecrets, "include-secrets", false, "Include launcher secrets (such as Kubernetes tokens or AWS secret keys) in the output") } func runPoolsExport(cmd *cobra.Command, args []string) error { @@ -1053,7 +1034,7 @@ func runPoolsExport(cmd *cobra.Command, args []string) error { return err } - result, err := client.GetPoolConfigs(cmd.Context(), workspaceID, poolsExportIncludeSecrets) + result, err := client.GetPoolConfigs(cmd.Context(), workspaceID) if err != nil { return err } @@ -1094,13 +1075,6 @@ func runPoolsExport(cmd *cobra.Command, args []string) error { fmt.Print(buf.String()) } - // Better to hear this now than when an import is refused later. - if names := redactedPools(pools); len(names) > 0 { - fmt.Fprintf(os.Stderr, - "Warning: secrets were redacted for %s, so this is not a complete configuration and cannot be imported as-is.\nRe-run with --include-secrets to export it in full.\n", - strings.Join(names, ", ")) - } - return nil } @@ -1177,12 +1151,6 @@ func runPoolsImport(cmd *cobra.Command, args []string) error { desiredPools[name] = tomlPoolToAPI(pool) } - if names := redactedPools(desiredPools); len(names) > 0 { - return fmt.Errorf( - "%s: secrets were redacted when this was exported, so importing it would clear them.\nRe-export with --include-secrets, or set the secret explicitly with: coflux pools update --set =", - strings.Join(names, ", ")) - } - // Connect and get current state workspace, err := requireWorkspace() if err != nil { @@ -1199,9 +1167,9 @@ func runPoolsImport(cmd *cobra.Command, args []string) error { return err } - // With secrets: out-of-scope pools are merged into the update below, so - // redacted values here would overwrite the real ones on the server. - result, err := client.GetPoolConfigs(cmd.Context(), workspaceID, true) + // Out-of-scope pools are merged into the update below, so this needs + // the full current configuration. + result, err := client.GetPoolConfigs(cmd.Context(), workspaceID) if err != nil { return err } @@ -1454,9 +1422,9 @@ var camelToSnake = map[string]string{ "securityGroups": "security_groups", "assignPublicIp": "assign_public_ip", "platformVersion": "platform_version", - "accessKeyId": "access_key_id", - "secretAccessKey": "secret_access_key", - "sessionToken": "session_token", + "tokenSecret": "token_secret", + "credentialsSecret": "credentials_secret", + "envSecrets": "env_secrets", } var snakeToCamel map[string]string @@ -1489,7 +1457,7 @@ func apiPoolToTOML(pool map[string]any) map[string]any { } } if idleTimeout, ok := pool["idleTimeout"]; ok { - result["idle_timeout"] = idleTimeout + result["idle_timeout"] = tomlNumber(idleTimeout) } if launcher, ok := pool["launcher"].(map[string]any); ok { result["launcher"] = apiLauncherToTOML(launcher) @@ -1504,17 +1472,27 @@ func apiLauncherToTOML(launcher map[string]any) map[string]any { if snakeKey, ok := camelToSnake[k]; ok { key = snakeKey } - if key == "env" { + if key == "env" || key == "env_secrets" { if m, ok := v.(map[string]any); ok { result[key] = inlineMap{m} continue } } - result[key] = v + result[key] = tomlNumber(v) } return result } +// tomlNumber keeps a whole number whole in TOML. JSON numbers arrive as +// float64, and every numeric pool field is an integer, so without this +// a concurrency of 8 would be exported as 8.0. +func tomlNumber(v any) any { + if f, ok := v.(float64); ok && f == float64(int64(f)) { + return int64(f) + } + return v +} + func tomlPoolToAPI(pool map[string]any) map[string]any { result := make(map[string]any) diff --git a/cli/cmd/coflux/secrets.go b/cli/cmd/coflux/secrets.go new file mode 100644 index 00000000..8e6b0a30 --- /dev/null +++ b/cli/cmd/coflux/secrets.go @@ -0,0 +1,236 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "sort" + + "github.com/spf13/cobra" +) + +var secretsCmd = &cobra.Command{ + Use: "secrets", + Short: "Manage secrets", + Long: `Manage secrets: values that pools refer to by name - a launcher's credentials, +or environment variables for workers - and that never appear in a pool's +configuration. + +A secret is set for a scope: a workspace name, or a prefix of one. A secret for +'development' applies to 'development/joe', and the nearest scope wins. Without +--scope or --global, the scope is the current workspace.`, +} + +var ( + secretsScope string + secretsGlobal bool + secretsFromEnv string + secretsFromFile string +) + +func init() { + for _, cmd := range []*cobra.Command{secretsSetCmd, secretsDeleteCmd} { + cmd.Flags().StringVar(&secretsScope, "scope", "", "Workspace name, or prefix, the secret applies to (default: the current workspace)") + cmd.Flags().BoolVar(&secretsGlobal, "global", false, "Apply to every workspace in the project") + } + secretsSetCmd.Flags().StringVar(&secretsFromEnv, "from-env", "", "Read the value from this environment variable") + secretsSetCmd.Flags().StringVar(&secretsFromFile, "from-file", "", "Read the value from this file") + secretsCmd.AddCommand(secretsSetCmd, secretsListCmd, secretsDeleteCmd) +} + +// secretScope is the scope a secret is set or deleted in: "" for the whole +// project, else a workspace name or prefix. +func secretScope() (string, error) { + if secretsGlobal && secretsScope != "" { + return "", fmt.Errorf("--scope and --global can't both be given") + } + if secretsGlobal { + return "", nil + } + if secretsScope != "" { + return secretsScope, nil + } + return requireWorkspace() +} + +func describeScope(scope string) string { + if scope == "" { + return "all workspaces" + } + return fmt.Sprintf("'%s'", scope) +} + +// secrets set + +var secretsSetCmd = &cobra.Command{ + Use: "set ", + Short: "Set a secret's value", + Long: `Set a secret's value, creating it or replacing what it had. + +The value is read from stdin, so it never appears on the command line or in +shell history: + + printf '%s' "$API_KEY" | coflux secrets set api-key + aws configure export-credentials --profile sandbox | coflux secrets set aws-sandbox + +Or from an environment variable (--from-env) or a file (--from-file). A single +trailing newline is dropped, so 'echo' works too.`, + Args: cobra.ExactArgs(1), + RunE: runSecretsSet, +} + +func runSecretsSet(cmd *cobra.Command, args []string) error { + name := args[0] + + scope, err := secretScope() + if err != nil { + return err + } + + value, err := readSecretValue() + if err != nil { + return err + } + + client, err := newClient() + if err != nil { + return err + } + + result, err := client.SetSecret(cmd.Context(), scope, name, value) + if err != nil { + return err + } + + fmt.Printf("Set secret '%s' (version %d) for %s.\n", result.Name, result.Version, describeScope(result.Scope)) + return nil +} + +func readSecretValue() (string, error) { + if secretsFromEnv != "" && secretsFromFile != "" { + return "", fmt.Errorf("--from-env and --from-file can't both be given") + } + + var data []byte + switch { + case secretsFromEnv != "": + value, ok := os.LookupEnv(secretsFromEnv) + if !ok { + return "", fmt.Errorf("environment variable %s is not set", secretsFromEnv) + } + return value, nil + case secretsFromFile != "": + read, err := os.ReadFile(secretsFromFile) + if err != nil { + return "", err + } + data = read + default: + if info, err := os.Stdin.Stat(); err == nil && info.Mode()&os.ModeCharDevice != 0 { + fmt.Fprintln(os.Stderr, "Enter the value, then press Ctrl-D:") + } + read, err := io.ReadAll(os.Stdin) + if err != nil { + return "", err + } + data = read + } + + data = bytes.TrimSuffix(data, []byte("\n")) + data = bytes.TrimSuffix(data, []byte("\r")) + if len(data) == 0 { + return "", fmt.Errorf("no value given") + } + return string(data), nil +} + +// secrets list + +var secretsListCmd = &cobra.Command{ + Use: "list", + Short: "List secrets", + Long: "List the project's secrets: their names, scopes and versions. Never their values.", + RunE: runSecretsList, +} + +func runSecretsList(cmd *cobra.Command, args []string) error { + client, err := newClient() + if err != nil { + return err + } + + secrets, err := client.ListSecrets(cmd.Context()) + if err != nil { + return err + } + + sort.Slice(secrets, func(i, j int) bool { + si, sj := getString(secrets[i], "scope"), getString(secrets[j], "scope") + if si != sj { + return si < sj + } + return getString(secrets[i], "name") < getString(secrets[j], "name") + }) + + if isOutput("json") { + return outputJSON(secrets) + } + + if len(secrets) == 0 { + fmt.Println("No secrets found.") + return nil + } + + var rows [][]string + for _, s := range secrets { + scope := getString(s, "scope") + if scope == "" { + scope = "(all)" + } + by := "-" + if principal, ok := s["updatedBy"].(map[string]any); ok { + by = fmt.Sprintf("%s %s", getString(principal, "type"), getString(principal, "externalId")) + } + rows = append(rows, []string{ + getString(s, "name"), + scope, + fmt.Sprintf("%d", int(getFloat64(s, "version"))), + formatTimestamp(getInt64(s, "updatedAt")), + by, + }) + } + + printTable([]string{"Name", "Scope", "Version", "Updated", "By"}, rows) + return nil +} + +// secrets delete + +var secretsDeleteCmd = &cobra.Command{ + Use: "delete ", + Short: "Delete a secret", + Args: cobra.ExactArgs(1), + RunE: runSecretsDelete, +} + +func runSecretsDelete(cmd *cobra.Command, args []string) error { + name := args[0] + + scope, err := secretScope() + if err != nil { + return err + } + + client, err := newClient() + if err != nil { + return err + } + + if err := client.DeleteSecret(cmd.Context(), scope, name); err != nil { + return err + } + + fmt.Printf("Deleted secret '%s' for %s.\n", name, describeScope(scope)) + return nil +} diff --git a/cli/internal/api/client.go b/cli/internal/api/client.go index 2a69427b..348a1090 100644 --- a/cli/internal/api/client.go +++ b/cli/internal/api/client.go @@ -260,9 +260,8 @@ type GetPoolConfigsResult struct { } // GetPoolConfigs retrieves all pool configs for a workspace along with an ETag. -// Secrets (such as a Kubernetes token) are redacted unless includeSecrets is set. -func (c *Client) GetPoolConfigs(ctx context.Context, workspaceID string, includeSecrets bool) (*GetPoolConfigsResult, error) { - body := map[string]any{"workspaceId": workspaceID, "includeSecrets": includeSecrets} +func (c *Client) GetPoolConfigs(ctx context.Context, workspaceID string) (*GetPoolConfigsResult, error) { + body := map[string]any{"workspaceId": workspaceID} var pools map[string]map[string]any headers, err := c.post(ctx, "/api/get_pools", body, &pools) if err != nil { @@ -368,6 +367,45 @@ func (c *Client) RevokeToken(ctx context.Context, externalID string) error { return err } +// Secrets API + +// SetSecretResult is what the server says about a secret it has set. +type SetSecretResult struct { + Name string `json:"name"` + Scope string `json:"scope"` + Version int `json:"version"` +} + +// SetSecret sets a secret's value for a scope ("" for the whole project). +func (c *Client) SetSecret(ctx context.Context, scope, name, value string) (*SetSecretResult, error) { + body := map[string]any{"name": name, "value": value, "scope": scope} + var result SetSecretResult + if _, err := c.post(ctx, "/api/set_secret", body, &result); err != nil { + return nil, err + } + return &result, nil +} + +// DeleteSecret deletes a secret from a scope ("" for the whole project). +func (c *Client) DeleteSecret(ctx context.Context, scope, name string) error { + body := map[string]any{"name": name, "scope": scope} + _, err := c.post(ctx, "/api/delete_secret", body, nil) + return err +} + +// ListSecrets lists the project's secrets: names, scopes and versions. +func (c *Client) ListSecrets(ctx context.Context) ([]map[string]any, error) { + var result map[string]map[string]any + if err := c.get(ctx, "/topics/secrets", nil, &result); err != nil { + return nil, err + } + secrets := make([]map[string]any, 0, len(result)) + for _, secret := range result { + secrets = append(secrets, secret) + } + return secrets, nil +} + // RerunStepResult contains the IDs returned from re-running a step type RerunStepResult struct { ExecutionID string `json:"executionId"` diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index 8974cabf..68361144 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -181,7 +181,7 @@ All manifest commands accept `--adapter` to specify the adapter command. | `pools disable ` | Disable a pool (drain workers) | | `pools enable ` | Re-enable a pool | | `pools launches [worker-id]` | View launched workers (`--watch`) | -| `pools export` | Export pool configs as TOML (`-o`, `--only`, `--include-secrets`) | +| `pools export` | Export pool configs as TOML (`-o`, `--only`) | | `pools import [file]` | Import pool configs from TOML or stdin. Declarative: pools absent from the file are deleted. Flags: `--dry-run`, `--yes`, `--only` | ### Pool creation flags @@ -214,6 +214,14 @@ See [pools](./pools.md) for launcher-specific fields. | `tokens create` | Create a token (`--name`, `--workspaces`) | | `tokens revoke ` | Revoke a token | +## `coflux secrets` + +| Command | Description | +|---------|-------------| +| `secrets set ` | Set a secret's value from stdin (`--from-env`, `--from-file`, `--scope`, `--global`) | +| `secrets list` | List secrets (names, scopes and versions) | +| `secrets delete ` | Delete a secret (`--scope`, `--global`) | + ## `coflux inputs` | Command | Description | diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 9e15e236..e28b45a7 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -68,25 +68,25 @@ coflux pools create mypool --type kubernetes \ --modules myapp.workflows ``` -When the Coflux server runs inside Kubernetes, it automatically uses in-cluster authentication. For external servers, provide the API server URL and a bearer token: +When the Coflux server runs inside Kubernetes, it automatically uses in-cluster authentication. For external servers, provide the API server URL and name a [secret](#secrets) holding a bearer token: ```bash +coflux secrets set k8s-token < /path/to/token + coflux pools create mypool --type kubernetes \ --set image=myorg/myapp:latest \ --set apiServer=https://my-cluster.example.com:6443 \ - --set token="$(cat /path/to/token)" \ + --set tokenSecret=k8s-token \ --set serverHost=coflux.example.com:7777 \ --modules myapp.workflows ``` -Note that the token is stored in the orchestration database. - | Field | Description | |-------|-------------| | `image` | Container image to run | | `namespace` | Kubernetes namespace (default: `default`) | | `apiServer` | Kubernetes API server URL (default: in-cluster) | -| `token` | Bearer token for API authentication | +| `tokenSecret` | Name of the secret holding the bearer token | | `caCert` | Path, on the server's host, to a CA certificate file for TLS verification | | `insecure` | Skip TLS verification | | `serviceAccount` | Service account for launched pods | @@ -132,14 +132,21 @@ Workers connect out to the server, so a task needs a route to it and nothing needs to reach the task: a public IP (`assignPublicIp`) in a public subnet, or a NAT gateway from a private one. -Credentials are taken from the pool when `accessKeyId` and -`secretAccessKey` are set (with `sessionToken` for temporary ones), and -otherwise from the server's surroundings the way the AWS SDKs look: -`AWS_ACCESS_KEY_ID` and friends in its environment, its ECS task role, -or its EC2 instance profile. They need `ecs:RunTask`, `ecs:DescribeTasks` -and `ecs:StopTask` on the cluster, `ecs:DescribeTaskDefinition` unless -`containerName` is set, and `iam:PassRole` for the roles the task -definition names. +Credentials come from the [secret](#secrets) named by `credentialsSecret`, +whose value is JSON in the shape the AWS CLI produces, so a profile's +credentials can be stored directly: + +```bash +aws configure export-credentials --profile sandbox | coflux secrets set aws-sandbox +coflux pools update mypool --set credentialsSecret=aws-sandbox +``` + +Without one, credentials come from the server's surroundings the way the +AWS SDKs look: `AWS_ACCESS_KEY_ID` and friends in its environment, its ECS +task role, or its EC2 instance profile. Either way they need `ecs:RunTask`, +`ecs:DescribeTasks` and `ecs:StopTask` on the cluster, +`ecs:DescribeTaskDefinition` unless `containerName` is set, and +`iam:PassRole` for the roles the task definition names. ECS doesn't expose container output through its API, so a worker's log tail isn't shown; a task that fails to start reports its reason in its @@ -158,9 +165,7 @@ see what workers print. | `securityGroups` | Security group IDs for the task | | `assignPublicIp` | Give the task a public IP | | `platformVersion` | Fargate platform version | -| `accessKeyId` | AWS access key ID | -| `secretAccessKey` | AWS secret access key | -| `sessionToken` | AWS session token, for temporary credentials | +| `credentialsSecret` | Name of the secret holding AWS credentials as JSON (`AccessKeyId`, `SecretAccessKey`, optional `SessionToken`) | | `endpoint` | ECS API endpoint override (e.g. a VPC endpoint) | ### Common fields @@ -178,6 +183,41 @@ These fields apply to all launcher types: | `adapter` | Adapter command | | `concurrency` | Maximum concurrent executions per worker | | `env` | Environment variables (e.g., `--set env.KEY=VALUE`) | +| `envSecrets` | Environment variables set from secrets (e.g., `--set envSecrets.API_KEY=api-key`) | + +## Secrets + +Anything a pool needs that mustn't be written down — an API key for workers, +a cluster token, cloud credentials — is a _secret_: stored by the server, +encrypted, and referred to by name. A pool's configuration, `pools get`, and +`pools export` only ever carry the name. + +The value is read from stdin, so it never appears on the command line: + +```bash +printf '%s' "$OPENAI_API_KEY" | coflux secrets set openai +coflux pools update mypool --set envSecrets.OPENAI_API_KEY=openai +``` + +`--from-env` and `--from-file` read it from an environment variable or a file +instead. `secrets list` shows names, scopes and versions, never values, and +`secrets delete` removes one. + +A secret applies to a _scope_: a workspace name, or a prefix of one. A secret +set for `development` applies to `development/joe`, and the nearest scope wins, +so `development/joe` can override it. By default the scope is the current +workspace; `--scope` names another, and `--global` applies it to every +workspace. Scopes follow workspace names, not what a workspace inherits from: +a workspace that inherits from `production` doesn't see production's secrets. + +Setting a secret takes operator access to its scope, and `--global` takes +access to every workspace. Setting one again replaces its value and bumps its +version; workers already running keep the value they were launched with. +Values are encrypted with a key derived from `COFLUX_SECRET`, which must be +configured for secrets to be used. + +A pool that names a secret its workspace can't see is refused when it is +created, updated, or imported. ## Managing pools @@ -222,14 +262,9 @@ coflux pools export --only mypool --only gpu-pool -o pools.toml coflux pools import pools.toml ``` -Launcher secrets — the Kubernetes `token`, and the ECS `secretAccessKey` and -`sessionToken` — are redacted on export unless `--include-secrets` is given. Importing a redacted file is refused rather -than silently clearing the secrets it omits, so use `--include-secrets` when the -exported file is meant to be imported again: - -```bash -coflux pools export --include-secrets -o pools.toml -``` +Pools refer to secrets by name, so an export never contains a value. Import it +somewhere else and the same secrets must exist there, or the import is refused +with the names that are missing. ## Provides, accepts, and requires diff --git a/docs/docs/server_config.md b/docs/docs/server_config.md index 67e7ccc8..84a741c5 100644 --- a/docs/docs/server_config.md +++ b/docs/docs/server_config.md @@ -24,7 +24,7 @@ This is a convenience wrapper around `docker run`. Docker must be installed and | `--data-dir` | `./data` | Directory for persistent data | | `--no-auth` | `false` | Disable authentication | | `--super-token` or `--super-token-hash` | _(none)_ | Set a super token (plain text or pre-hashed SHA-256 hex) | -| `--secret` | _(none)_ | Server secret for signing service tokens | +| `--secret` | _(none)_ | Server secret, for signing service tokens and encrypting secrets | | `--team` | _(none)_ | Team IDs allowed for Studio auth (repeatable) | | `--launcher` | _(none)_ | Allowed launcher types (repeatable: `docker`, `process`, `kubernetes`, `ecs`) | | `--image` | _(auto)_ | Docker image to use | @@ -65,7 +65,7 @@ The server is configured via environment variables. When using `coflux server`, | `COFLUX_PUBLIC_HOST` | `localhost:PORT` | Public host (use `%` prefix for subdomain routing) | | `COFLUX_REQUIRE_AUTH` | `true` | Whether authentication is required | | `COFLUX_SUPER_TOKEN_HASH` | _(none)_ | SHA-256 hex hash of the super token | -| `COFLUX_SECRET` | _(none)_ | Server secret for signing service tokens | +| `COFLUX_SECRET` | _(none)_ | Server secret, for signing service tokens and encrypting secrets | | `COFLUX_STUDIO_TEAMS` | _(none)_ | Comma-separated team IDs for Studio auth | | `COFLUX_STUDIO_URL` | `https://studio.coflux.com` | Studio URL | | `COFLUX_DATA_DIR` | `./data` | Data directory path | @@ -75,4 +75,4 @@ The server is configured via environment variables. When using `coflux server`, ## Data storage -The server stores data in the configured data directory. Each project gets its own set of SQLite databases. Orchestration, logs and metrics data is managed in rotating epochs, which allows the server to manage data growth without losing access to historical runs. An admin database holds service tokens, and isn't rotated. +The server stores data in the configured data directory. Each project gets its own set of SQLite databases. Orchestration, logs and metrics data is managed in rotating epochs, which allows the server to manage data growth without losing access to historical runs. An admin database holds service tokens and secrets, and isn't rotated. diff --git a/server/lib/coflux/admin/secrets.ex b/server/lib/coflux/admin/secrets.ex new file mode 100644 index 00000000..ed275988 --- /dev/null +++ b/server/lib/coflux/admin/secrets.ex @@ -0,0 +1,368 @@ +defmodule Coflux.Admin.Secrets do + @moduledoc """ + Secrets, in the admin store: values a pool needs that must never appear + in its configuration - API keys for workers, credentials for launchers. + + Each is encrypted with a key derived from the server secret, and bound + to what it is: a ciphertext moved to another row won't decrypt. There's + one current value per scope and name. Setting it again replaces the + value and bumps the version; deleting it removes it. Nothing older is + kept, so a rotated or deleted value is gone from here. + + A secret is scoped to a workspace name, or to a prefix of one: a secret + for `development` applies to `development/joe`, and the nearest scope + wins. The empty scope is the whole project. This follows the naming + hierarchy rather than the base-workspace chain: a workspace that + inherits results from `production` doesn't inherit its secrets. + + Pools name secrets in fields that take nothing else - `tokenSecret`, + `credentialsSecret`, `envSecrets` - and the values are resolved for a + launcher just before it is called, never stored with it. + """ + + alias Coflux.Store + + @key_id "hkdf-sha256-v1" + @nonce_bytes 12 + @tag_bytes 16 + @max_value_bytes 65_536 + + @name_regex ~r/^[a-z0-9][a-z0-9_-]{0,63}$/i + @scope_regex ~r/^[a-z0-9][a-z0-9_\/-]{0,99}$/i + + def valid_name?(name), do: is_binary(name) and Regex.match?(@name_regex, name) + + def valid_scope?(""), do: true + + def valid_scope?(scope), + do: + is_binary(scope) and Regex.match?(@scope_regex, scope) and + not String.ends_with?(scope, "/") + + def valid_value?(value), do: is_binary(value) and byte_size(value) <= @max_value_bytes + + @doc """ + Sets a secret's value, replacing whatever it had. `updated_by` is an + identity as `Principals.build/2` describes it, or nil. + + Returns `{:ok, secret}` with the secret's metadata (never its value), or + `{:error, :no_secret}` when the server has no secret to encrypt with. + """ + def set(db, project_id, scope, name, value, updated_by) do + with {:ok, key} <- encryption_key() do + now = System.system_time(:second) + + {existing_id, created_at, version} = + case Store.query_one( + db, + "SELECT id, created_at, version FROM secrets WHERE scope = ?1 AND name = ?2", + {scope, name} + ) do + {:ok, {id, created_at, version}} -> {id, created_at, version + 1} + {:ok, nil} -> {nil, now, 1} + end + + nonce = :crypto.strong_rand_bytes(@nonce_bytes) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_256_gcm, + key, + nonce, + value, + aad(project_id, scope, name, version), + @tag_bytes, + true + ) + + updated_by_type = updated_by && updated_by.type + updated_by_external_id = updated_by && updated_by.external_id + + if existing_id do + {:ok, _} = + Store.query( + db, + """ + UPDATE secrets + SET version = ?1, key_id = ?2, nonce = ?3, ciphertext = ?4, updated_at = ?5, + updated_by_type = ?6, updated_by_external_id = ?7 + WHERE id = ?8 + """, + {version, @key_id, {:blob, nonce}, {:blob, ciphertext <> tag}, now, updated_by_type, + updated_by_external_id, existing_id} + ) + else + {:ok, _} = + Store.insert_one(db, :secrets, %{ + scope: scope, + name: name, + version: version, + key_id: @key_id, + nonce: {:blob, nonce}, + ciphertext: {:blob, ciphertext <> tag}, + created_at: created_at, + updated_at: now, + updated_by_type: updated_by_type, + updated_by_external_id: updated_by_external_id + }) + end + + {:ok, + %{ + scope: scope, + name: name, + version: version, + created_at: created_at, + updated_at: now, + updated_by: updated_by + }} + end + end + + def delete(db, scope, name) do + case Store.query_one( + db, + "SELECT id FROM secrets WHERE scope = ?1 AND name = ?2", + {scope, name} + ) do + {:ok, {id}} -> + {:ok, _} = Store.query(db, "DELETE FROM secrets WHERE id = ?1", {id}) + :ok + + {:ok, nil} -> + {:error, :not_found} + end + end + + @doc "Every secret's metadata - never a value." + def list(db) do + {:ok, rows} = + Store.query( + db, + """ + SELECT scope, name, version, created_at, updated_at, updated_by_type, updated_by_external_id + FROM secrets + ORDER BY scope, name + """, + {} + ) + + {:ok, + Enum.map(rows, fn {scope, name, version, created_at, updated_at, type, external_id} -> + %{ + scope: scope, + name: name, + version: version, + created_at: created_at, + updated_at: updated_at, + updated_by: if(type, do: %{type: type, external_id: external_id}) + } + end)} + end + + @doc """ + The value of a secret as seen from a workspace: the one in the nearest + scope that covers the workspace. + + Returns `{:ok, value}`, `{:error, :not_found}`, `{:error, :undecryptable}` + (the server secret isn't the one it was encrypted with, or the row was + tampered with), or `{:error, :no_secret}`. + """ + def resolve(db, project_id, workspace_name, name) do + with {:ok, key} <- encryption_key(), + {:ok, {scope, version, nonce, blob}} <- find(db, workspace_name, name) do + decrypt(key, project_id, scope, name, version, nonce, blob) + end + end + + def exists?(db, workspace_name, name), do: match?({:ok, _}, find(db, workspace_name, name)) + + @doc "Whether a secret in `scope` is one a workspace of this name sees." + def scope_applies?("", _workspace_name), do: true + + def scope_applies?(scope, workspace_name), + do: workspace_name == scope or String.starts_with?(workspace_name, scope <> "/") + + # --- Launcher configs --- + + @doc "The names of the secrets a launcher config refers to." + def references(launcher) when is_map(launcher) do + env_secrets = Map.get(launcher, :env_secrets) || %{} + + [Map.get(launcher, :token_secret), Map.get(launcher, :credentials_secret)] + |> Enum.concat(Map.values(env_secrets)) + |> Enum.filter(&is_binary/1) + |> Enum.uniq() + end + + def references(_launcher), do: [] + + @doc """ + Checks that every secret a launcher config refers to exists for a + workspace, so a pool that could never launch is refused when it is + saved rather than found out when it is used. + """ + def check_references(db, workspace_name, launcher) do + case Enum.reject(references(launcher), &exists?(db, workspace_name, &1)) do + [] -> :ok + missing -> {:error, {:secrets_not_found, missing}} + end + end + + @doc """ + Gives a launcher config the values of the secrets it refers to, for the + launcher to use and then forget: `token_secret` becomes `token`, + `credentials_secret` becomes the AWS key fields, and `env_secrets` are + merged into `env`. + + Returns `{:ok, launcher}`, `{:error, {:secret_not_found, name}}`, + `{:error, {:secret_invalid, name}}` (it can't be decrypted, or isn't in + the shape the field needs), or `{:error, :no_secret}`. + """ + def resolve_launcher(db, project_id, workspace_name, launcher) do + with {:ok, launcher} <- + resolve_field(db, project_id, workspace_name, launcher, :token_secret, fn l, value -> + {:ok, Map.put(l, :token, value)} + end), + {:ok, launcher} <- + resolve_field( + db, + project_id, + workspace_name, + launcher, + :credentials_secret, + &put_aws_credentials/2 + ) do + resolve_env(db, project_id, workspace_name, launcher) + end + end + + defp resolve_field(db, project_id, workspace_name, launcher, key, put) do + case Map.get(launcher, key) do + name when is_binary(name) -> + with {:ok, value} <- resolve_named(db, project_id, workspace_name, name), + {:ok, launcher} <- put_or_invalid(put.(launcher, value), name) do + {:ok, launcher} + end + + _ -> + {:ok, launcher} + end + end + + defp resolve_env(db, project_id, workspace_name, launcher) do + env_secrets = Map.get(launcher, :env_secrets) || %{} + + Enum.reduce_while(env_secrets, {:ok, launcher}, fn {var, name}, {:ok, launcher} -> + case resolve_named(db, project_id, workspace_name, name) do + {:ok, value} -> + env = Map.put(Map.get(launcher, :env) || %{}, var, value) + {:cont, {:ok, Map.put(launcher, :env, env)}} + + {:error, _} = error -> + {:halt, error} + end + end) + end + + defp resolve_named(db, project_id, workspace_name, name) do + case resolve(db, project_id, workspace_name, name) do + {:ok, value} -> {:ok, value} + {:error, :not_found} -> {:error, {:secret_not_found, name}} + {:error, :undecryptable} -> {:error, {:secret_invalid, name}} + {:error, :no_secret} -> {:error, :no_secret} + end + end + + defp put_or_invalid({:ok, launcher}, _name), do: {:ok, launcher} + defp put_or_invalid(:error, name), do: {:error, {:secret_invalid, name}} + + # AWS credentials as JSON, in the shape `aws configure export-credentials` + # and credential processes produce (`AccessKeyId`, `SecretAccessKey`, + # `SessionToken`), or the same keys in camelCase. + defp put_aws_credentials(launcher, value) do + with {:ok, %{} = creds} <- Jason.decode(value), + access_key_id when is_binary(access_key_id) <- + creds["AccessKeyId"] || creds["accessKeyId"], + secret_access_key when is_binary(secret_access_key) <- + creds["SecretAccessKey"] || creds["secretAccessKey"] do + launcher = + launcher + |> Map.put(:access_key_id, access_key_id) + |> Map.put(:secret_access_key, secret_access_key) + + case creds["SessionToken"] || creds["sessionToken"] do + token when is_binary(token) and token != "" -> + {:ok, Map.put(launcher, :session_token, token)} + + _ -> + {:ok, launcher} + end + else + _ -> :error + end + end + + # --- Storage and crypto --- + + # Nearest scope wins: the longest that covers the workspace. + defp find(db, workspace_name, name) do + {:ok, rows} = + Store.query( + db, + "SELECT scope, version, nonce, ciphertext FROM secrets WHERE name = ?1", + {name} + ) + + rows + |> Enum.filter(fn {scope, _, _, _} -> scope_applies?(scope, workspace_name) end) + |> Enum.max_by(fn {scope, _, _, _} -> byte_size(scope) end, fn -> nil end) + |> case do + nil -> {:error, :not_found} + row -> {:ok, row} + end + end + + defp decrypt(key, project_id, scope, name, version, nonce, blob) + when byte_size(blob) >= @tag_bytes do + ciphertext_size = byte_size(blob) - @tag_bytes + <> = blob + + case :crypto.crypto_one_time_aead( + :aes_256_gcm, + key, + nonce, + ciphertext, + aad(project_id, scope, name, version), + tag, + false + ) do + :error -> {:error, :undecryptable} + value -> {:ok, value} + end + end + + defp decrypt(_key, _project_id, _scope, _name, _version, _nonce, _blob), + do: {:error, :undecryptable} + + # Binding the ciphertext to its row means the row can't be repurposed: + # a value can't be moved to another name, scope, or project. + defp aad(project_id, scope, name, version), + do: Enum.join(["coflux", "secret", project_id, scope, name, Integer.to_string(version)], "\n") + + # The server secret is the root; the key is derived from it for this + # purpose alone, so it is never the same bytes that sign anything. + defp encryption_key do + case Coflux.Config.secret() do + nil -> {:error, :no_secret} + secret -> {:ok, hkdf_sha256(secret, "coflux-secrets", "aes-256-gcm", 32)} + end + end + + # RFC 5869, for outputs of one hash block or less. + defp hkdf_sha256(ikm, salt, info, length) when length <= 32 do + prk = :crypto.mac(:hmac, :sha256, salt, ikm) + okm = :crypto.mac(:hmac, :sha256, prk, info <> <<1>>) + binary_part(okm, 0, length) + end +end diff --git a/server/lib/coflux/application.ex b/server/lib/coflux/application.ex index f42e19c6..9c523649 100644 --- a/server/lib/coflux/application.ex +++ b/server/lib/coflux/application.ex @@ -49,6 +49,7 @@ defmodule Coflux.Application do Topics.Search, Topics.Manifests, Topics.Tokens, + Topics.Secrets, Topics.Asset, Topics.Catalog, Topics.Queue, diff --git a/server/lib/coflux/events.ex b/server/lib/coflux/events.ex index 7009ca92..eb4b7aae 100644 --- a/server/lib/coflux/events.ex +++ b/server/lib/coflux/events.ex @@ -316,6 +316,19 @@ defmodule Coflux.Events.TokenRevoked do defstruct [:token] end +defmodule Coflux.Events.SecretSet do + @moduledoc """ + Row: `secrets` (admin store), created or replaced. Its value is not an + event: nothing that carries this ever sees it. + """ + defstruct [:scope, :name, :version, :created_at, :updated_at, :updated_by] +end + +defmodule Coflux.Events.SecretDeleted do + @moduledoc "Row: `secrets` (admin store), gone." + defstruct [:scope, :name] +end + defmodule Coflux.Events.WorkspaceCreated do @moduledoc "Row: `workspaces`. `base` is the base workspace's external id, or nil." defstruct [:workspace, :name, :base, :state] diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 34199c23..d9480ed7 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -6,11 +6,6 @@ defmodule Coflux.Handlers.Api do @max_parameters 20 - # Stands in for a secret the caller didn't ask for. It is deliberately - # not a valid value: a config exported without secrets and imported - # again is refused rather than quietly clearing the real one. - @redacted_secret "" - @ecs_launch_types ["FARGATE", "EC2", "EXTERNAL"] # A directory upload arrives as one entry per file, so this bounds an @@ -274,6 +269,7 @@ defmodule Coflux.Handlers.Api do ) do :ok -> :cowboy_req.reply(204, req) {:error, :already_exists} -> json_error_response(req, "already_exists", status: 409) + {:error, {:secrets_not_found, names}} -> secrets_not_found_response(req, names) {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) {:error, :workspace_invalid} -> json_error_response(req, "not_found", status: 404) end @@ -300,6 +296,7 @@ defmodule Coflux.Handlers.Api do :ok -> :cowboy_req.reply(204, req) {:error, :not_found} -> json_error_response(req, "not_found", status: 404) {:error, :type_change} -> json_error_response(req, "type_change", status: 409) + {:error, {:secrets_not_found, names}} -> secrets_not_found_response(req, names) {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) {:error, :workspace_invalid} -> json_error_response(req, "not_found", status: 404) end @@ -356,19 +353,13 @@ defmodule Coflux.Handlers.Api do end defp handle(req, "POST", ["get_pools"], project_id, _access) do - case read_arguments( - req, - %{workspace_id: "workspaceId"}, - %{include_secrets: {"includeSecrets", &parse_boolean(&1, optional: true)}} - ) do + case read_arguments(req, %{workspace_id: "workspaceId"}) do {:ok, arguments, req} -> - include_secrets = Map.get(arguments, :include_secrets) == true - case Orchestration.get_pools(project_id, arguments.workspace_id) do {:ok, pools, hash} -> result = Map.new(pools, fn {name, pool} -> - {name, build_pool_config(pool, include_secrets)} + {name, build_pool_config(pool)} end) req = :cowboy_req.set_resp_header("etag", "\"#{hash}\"", req) @@ -405,6 +396,9 @@ defmodule Coflux.Handlers.Api do :ok -> :cowboy_req.reply(204, req) + {:error, {:secrets_not_found, names}} -> + secrets_not_found_response(req, names) + {:error, :conflict} -> json_error_response(req, "conflict", status: 412, @@ -1014,6 +1008,67 @@ defmodule Coflux.Handlers.Api do end end + defp handle(req, "POST", ["set_secret"], project_id, access) do + case read_arguments( + req, + %{ + name: {"name", &parse_secret_name/1}, + value: {"value", &parse_secret_value/1} + }, + %{scope: {"scope", &parse_secret_scope/1}} + ) do + {:ok, arguments, req} -> + case Orchestration.set_secret( + project_id, + Map.get(arguments, :scope, ""), + arguments.name, + arguments.value, + access + ) do + {:ok, secret} -> + json_response(req, %{ + "name" => secret.name, + "scope" => secret.scope, + "version" => secret.version + }) + + {:error, :forbidden} -> + json_error_response(req, "forbidden", status: 403) + + {:error, :no_secret} -> + json_error_response(req, "bad_request", + details: %{message: "Secrets require COFLUX_SECRET to be configured"} + ) + end + + {:error, errors, req} -> + json_error_response(req, "bad_request", details: errors) + end + end + + defp handle(req, "POST", ["delete_secret"], project_id, access) do + case read_arguments( + req, + %{name: {"name", &parse_secret_name/1}}, + %{scope: {"scope", &parse_secret_scope/1}} + ) do + {:ok, arguments, req} -> + case Orchestration.delete_secret( + project_id, + Map.get(arguments, :scope, ""), + arguments.name, + access + ) do + :ok -> :cowboy_req.reply(204, req) + {:error, :not_found} -> json_error_response(req, "not_found", status: 404) + {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) + end + + {:error, errors, req} -> + json_error_response(req, "bad_request", details: errors) + end + end + defp handle(req, "POST", ["revoke_token"], project_id, access) do case read_arguments(req, %{external_id: "externalId"}) do {:ok, arguments, req} -> @@ -1222,7 +1277,7 @@ defmodule Coflux.Handlers.Api do namespace = Map.get(value, "namespace") service_account = Map.get(value, "serviceAccount") api_server = Map.get(value, "apiServer") - token = Map.get(value, "token") + token_secret = Map.get(value, "tokenSecret") ca_cert = Map.get(value, "caCert") insecure = Map.get(value, "insecure") image_pull_policy = Map.get(value, "imagePullPolicy") @@ -1253,12 +1308,9 @@ defmodule Coflux.Handlers.Api do not is_nil(api_server) and (not is_binary(api_server) or String.length(api_server) > 500) -> {:error, :invalid} - not is_nil(token) and not is_binary(token) -> + not is_nil(token_secret) and not Coflux.Admin.Secrets.valid_name?(token_secret) -> {:error, :invalid} - token == @redacted_secret -> - {:error, :redacted} - not is_nil(ca_cert) and not is_binary(ca_cert) -> {:error, :invalid} @@ -1319,7 +1371,9 @@ defmodule Coflux.Handlers.Api do launcher = if api_server, do: Map.put(launcher, :api_server, api_server), else: launcher - launcher = if token, do: Map.put(launcher, :token, token), else: launcher + launcher = + if token_secret, do: Map.put(launcher, :token_secret, token_secret), else: launcher + launcher = if ca_cert, do: Map.put(launcher, :ca_cert, ca_cert), else: launcher launcher = @@ -1381,9 +1435,7 @@ defmodule Coflux.Handlers.Api do security_groups = wrap_list(Map.get(value, "securityGroups")) assign_public_ip = Map.get(value, "assignPublicIp") platform_version = Map.get(value, "platformVersion") - access_key_id = Map.get(value, "accessKeyId") - secret_access_key = Map.get(value, "secretAccessKey") - session_token = Map.get(value, "sessionToken") + credentials_secret = Map.get(value, "credentialsSecret") endpoint = Map.get(value, "endpoint") cond do @@ -1425,28 +1477,8 @@ defmodule Coflux.Handlers.Api do (not is_binary(platform_version) or String.length(platform_version) > 50) -> {:error, :invalid} - not is_nil(access_key_id) and - (not is_binary(access_key_id) or String.length(access_key_id) > 128) -> - {:error, :invalid} - - not is_nil(secret_access_key) and not is_binary(secret_access_key) -> - {:error, :invalid} - - secret_access_key == @redacted_secret -> - {:error, :redacted} - - not is_nil(session_token) and not is_binary(session_token) -> - {:error, :invalid} - - session_token == @redacted_secret -> - {:error, :redacted} - - # A key ID without its secret (or the reverse) can't sign anything, - # and a session token belongs to a key pair. - is_nil(access_key_id) != is_nil(secret_access_key) -> - {:error, :invalid} - - not is_nil(session_token) and is_nil(access_key_id) -> + not is_nil(credentials_secret) and + not Coflux.Admin.Secrets.valid_name?(credentials_secret) -> {:error, :invalid} not is_nil(endpoint) and @@ -1464,9 +1496,7 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value(:security_groups, security_groups) |> maybe_put_value(:assign_public_ip, if(assign_public_ip == true, do: true)) |> maybe_put_value(:platform_version, platform_version) - |> maybe_put_value(:access_key_id, access_key_id) - |> maybe_put_value(:secret_access_key, secret_access_key) - |> maybe_put_value(:session_token, session_token) + |> maybe_put_value(:credentials_secret, credentials_secret) |> maybe_put_value(:endpoint, endpoint) {:ok, launcher} @@ -1489,6 +1519,7 @@ defmodule Coflux.Handlers.Api do adapter = Map.get(value, "adapter") concurrency = Map.get(value, "concurrency") env = Map.get(value, "env") + env_secrets = Map.get(value, "envSecrets") cond do not is_nil(server_host) and (not is_binary(server_host) or String.length(server_host) > 200) -> @@ -1514,6 +1545,16 @@ defmodule Coflux.Handlers.Api do end) -> {:error, :invalid} + not is_nil(env_secrets) and not is_map(env_secrets) -> + {:error, :invalid} + + not is_nil(env_secrets) and + Enum.any?(env_secrets, fn {k, v} -> + not is_binary(k) or String.starts_with?(k, "COFLUX_") or + not Coflux.Admin.Secrets.valid_name?(v) + end) -> + {:error, :invalid} + true -> launcher = if server_host, do: Map.put(launcher, :server_host, server_host), else: launcher @@ -1529,6 +1570,10 @@ defmodule Coflux.Handlers.Api do if concurrency, do: Map.put(launcher, :concurrency, concurrency), else: launcher launcher = if env, do: Map.put(launcher, :env, env), else: launcher + + launcher = + if env_secrets, do: Map.put(launcher, :env_secrets, env_secrets), else: launcher + {:ok, launcher} end end @@ -1582,8 +1627,7 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(result, name, pool)}} # Keep why, against the pool it came from: "invalid" alone - # leaves the caller no idea which pool, or what to do about - # it - and `redacted` in particular has a specific remedy. + # leaves the caller no idea which pool. {:error, error} -> {:halt, {:error, %{name => error}}} @@ -1601,11 +1645,7 @@ defmodule Coflux.Handlers.Api do end end - defp secret_value(nil, _include_secrets), do: nil - defp secret_value(value, true), do: value - defp secret_value(_value, false), do: @redacted_secret - - defp build_pool_config(pool, include_secrets) do + defp build_pool_config(pool) do provides = pool.provides accepts = Map.get(pool, :accepts, %{}) @@ -1616,13 +1656,13 @@ defmodule Coflux.Handlers.Api do config = maybe_put_value(config, "idleTimeout", Map.get(pool, :idle_timeout)) if pool.launcher do - Map.put(config, "launcher", build_launcher_config(pool.launcher, include_secrets)) + Map.put(config, "launcher", build_launcher_config(pool.launcher)) else config end end - defp build_launcher_config(launcher, include_secrets) do + defp build_launcher_config(launcher) do type_fields = case launcher.type do :docker -> @@ -1647,15 +1687,7 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("securityGroups", Map.get(launcher, :security_groups)) |> maybe_put_value("assignPublicIp", Map.get(launcher, :assign_public_ip)) |> maybe_put_value("platformVersion", Map.get(launcher, :platform_version)) - |> maybe_put_value("accessKeyId", Map.get(launcher, :access_key_id)) - |> maybe_put_value( - "secretAccessKey", - secret_value(Map.get(launcher, :secret_access_key), include_secrets) - ) - |> maybe_put_value( - "sessionToken", - secret_value(Map.get(launcher, :session_token), include_secrets) - ) + |> maybe_put_value("credentialsSecret", Map.get(launcher, :credentials_secret)) |> maybe_put_value("endpoint", Map.get(launcher, :endpoint)) :kubernetes -> @@ -1663,7 +1695,7 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("namespace", Map.get(launcher, :namespace)) |> maybe_put_value("apiServer", Map.get(launcher, :api_server)) |> maybe_put_value("serviceAccount", Map.get(launcher, :service_account)) - |> maybe_put_value("token", secret_value(Map.get(launcher, :token), include_secrets)) + |> maybe_put_value("tokenSecret", Map.get(launcher, :token_secret)) |> maybe_put_value("caCert", Map.get(launcher, :ca_cert)) |> maybe_put_value("insecure", Map.get(launcher, :insecure)) |> maybe_put_value("imagePullPolicy", Map.get(launcher, :image_pull_policy)) @@ -1685,6 +1717,23 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("adapter", Map.get(launcher, :adapter)) |> maybe_put_value("concurrency", Map.get(launcher, :concurrency)) |> maybe_put_value("env", Map.get(launcher, :env)) + |> maybe_put_value("envSecrets", Map.get(launcher, :env_secrets)) + end + + defp parse_secret_name(value) do + if Coflux.Admin.Secrets.valid_name?(value), do: {:ok, value}, else: {:error, :invalid} + end + + defp parse_secret_scope(value) do + if Coflux.Admin.Secrets.valid_scope?(value), do: {:ok, value}, else: {:error, :invalid} + end + + defp parse_secret_value(value) do + if Coflux.Admin.Secrets.valid_value?(value), do: {:ok, value}, else: {:error, :invalid} + end + + defp secrets_not_found_response(req, names) do + json_error_response(req, "secrets_not_found", details: %{"secrets" => names}) end defp maybe_put_value(map, _key, nil), do: map @@ -1810,7 +1859,7 @@ defmodule Coflux.Handlers.Api do {"namespace", &is_binary/1}, {"serviceAccount", &is_binary/1}, {"apiServer", &is_binary/1}, - {"token", &is_binary/1}, + {"tokenSecret", &Coflux.Admin.Secrets.valid_name?/1}, {"caCert", &is_binary/1}, {"insecure", &is_boolean/1}, {"imagePullPolicy", &(&1 in valid_pull_policies)}, @@ -1829,9 +1878,7 @@ defmodule Coflux.Handlers.Api do {"securityGroups", &(is_binary(&1) or is_string_list?(&1, 5))}, {"assignPublicIp", &is_boolean/1}, {"platformVersion", &is_binary/1}, - {"accessKeyId", &is_binary/1}, - {"secretAccessKey", &is_binary/1}, - {"sessionToken", &is_binary/1}, + {"credentialsSecret", &Coflux.Admin.Secrets.valid_name?/1}, {"endpoint", &is_binary/1}, {"serverHost", &is_binary/1}, {"serverSecure", &is_boolean/1}, @@ -1844,6 +1891,14 @@ defmodule Coflux.Handlers.Api do is_binary(k) and (is_binary(val) or is_nil(val)) and not String.starts_with?(k, "COFLUX_") end) + end}, + {"envSecrets", + fn v -> + is_map(v) and + Enum.all?(v, fn {k, val} -> + is_binary(k) and (is_nil(val) or Coflux.Admin.Secrets.valid_name?(val)) and + not String.starts_with?(k, "COFLUX_") + end) end} ] @@ -1856,7 +1911,7 @@ defmodule Coflux.Handlers.Api do "namespace" => :namespace, "serviceAccount" => :service_account, "apiServer" => :api_server, - "token" => :token, + "tokenSecret" => :token_secret, "caCert" => :ca_cert, "insecure" => :insecure, "imagePullPolicy" => :image_pull_policy, @@ -1875,15 +1930,14 @@ defmodule Coflux.Handlers.Api do "securityGroups" => :security_groups, "assignPublicIp" => :assign_public_ip, "platformVersion" => :platform_version, - "accessKeyId" => :access_key_id, - "secretAccessKey" => :secret_access_key, - "sessionToken" => :session_token, + "credentialsSecret" => :credentials_secret, "endpoint" => :endpoint, "serverHost" => :server_host, "serverSecure" => :server_secure, "adapter" => :adapter, "concurrency" => :concurrency, - "env" => :env + "env" => :env, + "envSecrets" => :env_secrets } result = @@ -1899,7 +1953,7 @@ defmodule Coflux.Handlers.Api do if validator.(field_value) do processed_value = cond do - json_key == "env" and is_map(field_value) -> + json_key in ["env", "envSecrets"] and is_map(field_value) -> Map.new(field_value, fn {k, nil} -> {k, :unset} {k, v} -> {k, v} diff --git a/server/lib/coflux/launchers/docker.ex b/server/lib/coflux/launchers/docker.ex index 3d5076b9..cb529f1c 100644 --- a/server/lib/coflux/launchers/docker.ex +++ b/server/lib/coflux/launchers/docker.ex @@ -33,7 +33,7 @@ defmodule Coflux.DockerLauncher do end end - def stop(%{container: container_id, docker_conn: docker_conn}) do + def stop(%{container: container_id, docker_conn: docker_conn}, _config \\ nil) do case stop_container(docker_conn, container_id) do :ok -> case remove_container(docker_conn, container_id) do @@ -49,7 +49,7 @@ defmodule Coflux.DockerLauncher do end end - def poll(%{container: container_id, docker_conn: docker_conn}) do + def poll(%{container: container_id, docker_conn: docker_conn}, _config \\ nil) do case inspect_container(docker_conn, container_id) do {:ok, result} -> state = result["State"] diff --git a/server/lib/coflux/launchers/ecs.ex b/server/lib/coflux/launchers/ecs.ex index df709bd2..97323419 100644 --- a/server/lib/coflux/launchers/ecs.ex +++ b/server/lib/coflux/launchers/ecs.ex @@ -16,9 +16,8 @@ defmodule Coflux.EcsLauncher do tail. Credentials are resolved on every call rather than kept with the task: - the ones the server finds in its surroundings rotate, and the static - ones a pool can be configured with are the only kind that are safe to - remember. See `Coflux.Launchers.AwsCredentials`. + from the secret the pool names, or failing that from the server's own + surroundings. See `Coflux.Launchers.AwsCredentials`. """ import Coflux.Launchers.Utils, only: [truncate_bytes: 2] @@ -55,19 +54,12 @@ defmodule Coflux.EcsLauncher do with {:ok, conn} <- build_conn(config), {:ok, container_name} <- resolve_container_name(conn, config), {:ok, task_arn} <- run_task(conn, config, container_name, env, modules, opts) do - data = %{ - task_arn: task_arn, - cluster: Map.fetch!(config, :cluster), - container_name: container_name, - region: conn.region - } - - data = - data - |> maybe_put(:endpoint, config[:endpoint]) - |> maybe_put(:credentials, static_credentials(config)) - - {:ok, data} + {:ok, + %{ + task_arn: task_arn, + cluster: Map.fetch!(config, :cluster), + container_name: container_name + }} else {:error, reason} -> case normalize_launch_error(reason) do @@ -77,8 +69,8 @@ defmodule Coflux.EcsLauncher do end end - def stop(%{task_arn: task_arn, cluster: cluster} = data) do - with {:ok, conn} <- conn_from_data(data), + def stop(%{task_arn: task_arn, cluster: cluster}, config) do + with {:ok, conn} <- build_conn(config), {:ok, _body} <- ecs_request(conn, "StopTask", %{ "cluster" => cluster, @@ -97,8 +89,8 @@ defmodule Coflux.EcsLauncher do end end - def poll(%{task_arn: task_arn, cluster: cluster} = data) do - with {:ok, conn} <- conn_from_data(data), + def poll(%{task_arn: task_arn, cluster: cluster} = data, config) do + with {:ok, conn} <- build_conn(config), {:ok, body} <- ecs_request(conn, "DescribeTasks", %{"cluster" => cluster, "tasks" => [task_arn]}) do case body do @@ -372,19 +364,6 @@ defmodule Coflux.EcsLauncher do end end - defp conn_from_data(data) do - region = Map.fetch!(data, :region) - - with {:ok, credentials} <- AwsCredentials.resolve(data[:credentials]) do - {:ok, - %{ - region: region, - endpoint: data[:endpoint] || default_endpoint(region), - credentials: credentials - }} - end - end - defp default_endpoint(region), do: "https://ecs.#{region}.amazonaws.com" defp static_credentials( diff --git a/server/lib/coflux/launchers/kubernetes.ex b/server/lib/coflux/launchers/kubernetes.ex index ea0c3390..b0270c1c 100644 --- a/server/lib/coflux/launchers/kubernetes.ex +++ b/server/lib/coflux/launchers/kubernetes.ex @@ -1,4 +1,13 @@ defmodule Coflux.KubernetesLauncher do + @moduledoc """ + Runs workers as Kubernetes Jobs. + + What identifies a launched worker is its job's name and namespace. How + to reach the cluster comes from the pool's configuration on every call, + with the token resolved from the secret it names, so nothing that could + open the cluster is kept with the worker. + """ + import Coflux.Launchers.Utils, only: [truncate_bytes: 2] @log_tail_lines 20 @@ -88,14 +97,16 @@ defmodule Coflux.KubernetesLauncher do case k8s_request(conn, :post, path, json: job) do {:ok, %{"metadata" => %{"name" => name}}} -> - {:ok, %{job_name: name, namespace: namespace, k8s_conn: conn}} + {:ok, %{job_name: name, namespace: namespace}} {:error, reason} -> {:error, normalize_launch_error(reason)} end end - def stop(%{job_name: job_name, namespace: namespace, k8s_conn: conn}) do + def stop(%{job_name: job_name, namespace: namespace}, config) do + conn = build_conn(config) + path = "/apis/batch/v1/namespaces/#{namespace}/jobs/#{job_name}?propagationPolicy=Background" @@ -106,7 +117,8 @@ defmodule Coflux.KubernetesLauncher do end end - def poll(%{job_name: job_name, namespace: namespace, k8s_conn: conn}) do + def poll(%{job_name: job_name, namespace: namespace}, config) do + conn = build_conn(config) path = "/apis/batch/v1/namespaces/#{namespace}/jobs/#{job_name}" case k8s_request(conn, :get, path) do diff --git a/server/lib/coflux/launchers/process.ex b/server/lib/coflux/launchers/process.ex index 314d8003..796de70f 100644 --- a/server/lib/coflux/launchers/process.ex +++ b/server/lib/coflux/launchers/process.ex @@ -69,7 +69,7 @@ defmodule Coflux.ProcessLauncher do end end - def stop(data) do + def stop(data, _config \\ nil) do case resolve(data) do {:supervised, pid} -> GenServer.call(pid, :stop) @@ -85,7 +85,7 @@ defmodule Coflux.ProcessLauncher do end end - def poll(data) do + def poll(data, _config \\ nil) do case resolve(data) do {:supervised, pid} -> case GenServer.call(pid, :status) do diff --git a/server/lib/coflux/orchestration.ex b/server/lib/coflux/orchestration.ex index 09974a00..81158540 100644 --- a/server/lib/coflux/orchestration.ex +++ b/server/lib/coflux/orchestration.ex @@ -59,6 +59,14 @@ defmodule Coflux.Orchestration do call_server(project_id, {:archive_workspace, workspace_id, access}) end + def set_secret(project_id, scope, name, value, access \\ nil) do + call_server(project_id, {:set_secret, scope, name, value, access}) + end + + def delete_secret(project_id, scope, name, access \\ nil) do + call_server(project_id, {:delete_secret, scope, name, access}) + end + def get_pools(project_id, workspace_id) do call_server(project_id, {:get_pools, workspace_id}) end diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index d6c2cea0..b42c1810 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -18,6 +18,8 @@ defmodule Coflux.Orchestration.Server do StepArguments, StepCreated, StreamDependencyRecorded, + SecretDeleted, + SecretSet, TokenCreated, TokenRevoked, WorkspaceCreated, @@ -314,6 +316,44 @@ defmodule Coflux.Orchestration.Server do end end + # Secrets + + defp dispatch_call({:set_secret, scope, name, value, access}, state) do + with :ok <- check_secret_scope_access(access, scope), + {:ok, secret} <- + Coflux.Admin.Secrets.set( + state.admin_db, + state.project_id, + scope, + name, + value, + principal_identity(state, access) + ) do + state = + Effects.emit(state, %SecretSet{ + scope: secret.scope, + name: secret.name, + version: secret.version, + created_at: secret.created_at, + updated_at: secret.updated_at, + updated_by: secret.updated_by + }) + + {:reply, {:ok, secret}, state} + else + {:error, reason} -> {:reply, {:error, reason}, state} + end + end + + defp dispatch_call({:delete_secret, scope, name, access}, state) do + with :ok <- check_secret_scope_access(access, scope), + :ok <- Coflux.Admin.Secrets.delete(state.admin_db, scope, name) do + {:reply, :ok, Effects.emit(state, %SecretDeleted{scope: scope, name: name})} + else + {:error, reason} -> {:reply, {:error, reason}, state} + end + end + defp dispatch_call({:get_token, external_id}, state) do case Coflux.Admin.Tokens.get_token_by_external_id(state.admin_db, external_id) do {:ok, nil} -> @@ -547,7 +587,14 @@ defmodule Coflux.Orchestration.Server do state ) do with {:ok, workspace_id, _} <- - Permissions.require_workspace(state, workspace_external_id, access) do + Permissions.require_workspace(state, workspace_external_id, access), + :ok <- + Enum.reduce_while(desired_pools, :ok, fn {_name, pool}, :ok -> + case check_secret_references(state, workspace_id, pool[:launcher]) do + :ok -> {:cont, :ok} + error -> {:halt, error} + end + end) do case Workspaces.update_pools( state.db, workspace_id, @@ -610,7 +657,8 @@ defmodule Coflux.Orchestration.Server do state ) do with {:ok, workspace_id, _} <- - Permissions.require_workspace(state, workspace_external_id, access) do + Permissions.require_workspace(state, workspace_external_id, access), + :ok <- check_secret_references(state, workspace_id, pool[:launcher]) do case Workspaces.create_pool( state.db, workspace_id, @@ -646,7 +694,8 @@ defmodule Coflux.Orchestration.Server do state ) do with {:ok, workspace_id, _} <- - Permissions.require_workspace(state, workspace_external_id, access) do + Permissions.require_workspace(state, workspace_external_id, access), + :ok <- check_secret_references(state, workspace_id, pool_patch[:launcher]) do case Workspaces.update_pool( state.db, workspace_id, @@ -2927,6 +2976,35 @@ defmodule Coflux.Orchestration.Server do # Private helper functions + # A secret in a scope is a secret for every workspace under it, so + # setting one takes operator access to the scope itself, and the project + # scope takes access to everything. + defp check_secret_scope_access(nil, _scope), do: :ok + + defp check_secret_scope_access(access, "") do + if access[:workspaces] == :all, do: :ok, else: {:error, :forbidden} + end + + defp check_secret_scope_access(access, scope) do + if Permissions.operator?(access[:workspaces], scope), do: :ok, else: {:error, :forbidden} + end + + defp principal_identity(state, access) do + case Principals.get_principal(state.db, access && access[:principal_id]) do + {:ok, {type, external_id}} -> %{type: type, external_id: external_id} + {:ok, nil} -> nil + end + end + + # A pool naming a secret its workspace can't see would never launch, so + # it is refused now rather than found out then. + defp check_secret_references(state, workspace_id, launcher) when is_map(launcher) do + workspace_name = state.workspaces[workspace_id].name + Coflux.Admin.Secrets.check_references(state.admin_db, workspace_name, launcher) + end + + defp check_secret_references(_state, _workspace_id, _launcher), do: :ok + defp validate_values_assets(db, values) do Enum.reduce_while(values, :ok, fn value, :ok -> case validate_value_assets(db, value) do diff --git a/server/lib/coflux/orchestration/server/routing.ex b/server/lib/coflux/orchestration/server/routing.ex index a37cf75b..a51ca5d1 100644 --- a/server/lib/coflux/orchestration/server/routing.ex +++ b/server/lib/coflux/orchestration/server/routing.ex @@ -51,6 +51,8 @@ defmodule Coflux.Orchestration.Server.Routing do StreamDependencyRecorded, StreamItemAppended, StreamRegistered, + SecretDeleted, + SecretSet, TokenCreated, TokenRevoked, WorkerCreated, @@ -244,6 +246,8 @@ defmodule Coflux.Orchestration.Server.Routing do def route(%WorkspaceStateChanged{}, _state), do: [:workspaces] def route(%TokenCreated{}, _state), do: [:tokens] def route(%TokenRevoked{}, _state), do: [:tokens] + def route(%SecretSet{}, _state), do: [:secrets] + def route(%SecretDeleted{}, _state), do: [:secrets] # --------------------------------------------------------------------------- diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index a2b94cac..006c6f5c 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -514,72 +514,6 @@ defmodule Coflux.Orchestration.Server.Scheduler do |> put_in([Access.key(:session_ids), external_id], session_id) |> Fleet.schedule_session_expiry(session_id, activation_timeout) |> Listeners.maybe_schedule_idle_shutdown() - |> Fleet.call_launcher( - pool.launcher, - :launch, - [ - Fleet.build_launcher_env(state, workspace_id, token, pool.launcher), - pool.modules, - pool.launcher, - %{pool_name: pool_name} - ], - fn state, result -> - # A launcher can say more than a code about why a - # launch failed - an API's own message, typically - - # and that goes where a log tail would. - {data, error, detail} = - case result do - {:ok, {:ok, data}} -> {data, nil, nil} - {:ok, {:error, error}} -> {nil, error, nil} - {:ok, {:error, error, detail}} -> {nil, error, detail} - :error -> {nil, "launch_crashed", nil} - end - - {:ok, started_at} = - Workers.create_worker_launch_result(state.db, worker_id, data, error) - - state = - Effects.emit(state, %WorkerLaunchResult{ - workspace: State.workspace_external_id(state, workspace_id), - pool: pool_name, - worker: worker_external_id, - started_at: started_at, - error: error - }) - - cond do - error -> - # Deactivating the worker pops it from state, and - # with it the only record that this pool was ever - # tried - so count the failure first, or the pool - # relaunches on the very next pass. - state - |> record_pool_launch_failure(pool_id) - |> Fleet.deactivate_worker(worker_id, error, detail) - - Map.has_key?(state.workers, worker_id) -> - put_in( - state, - [Access.key(:workers), worker_id, Access.key(:data)], - data - ) - - true -> - # The worker was deactivated while its launch was - # in flight. Nothing will ever connect to what was - # just started, and this result is the only thing - # that knows how to reach it, so stop it here - # rather than leaking it. - Fleet.call_launcher( - state, - pool.launcher, - :stop, - [data], - fn state, _result -> state end - ) - end - end - ) |> put_in([Access.key(:workers), worker_id], %{ external_id: worker_external_id, created_at: created_at, @@ -604,6 +538,15 @@ defmodule Coflux.Orchestration.Server.Scheduler do created_at: created_at, session: external_id }) + |> start_launch( + workspace_id, + pool_id, + pool_name, + pool, + worker_id, + worker_external_id, + token + ) end end) @@ -622,29 +565,37 @@ defmodule Coflux.Orchestration.Server.Scheduler do state.workers |> Enum.filter(fn {_worker_id, worker} -> poll_due?(state, worker, now) end) |> Enum.reduce(state, fn {worker_id, worker}, state -> - {:ok, launcher} = Workspaces.get_launcher_for_pool(state.db, worker.pool_id) - - state - |> Fleet.call_launcher(launcher, :poll, [worker.data], fn state, result -> - state = update_worker(state, worker_id, &%{&1 | polling: false}) - - case result do - {:ok, {:ok, true}} -> - clear_poll_failures(state, worker_id) - - {:ok, {:ok, false, error, logs}} -> - # The launcher knows the worker has gone, and this is the - # only place its exit code and log tail come from. - Fleet.deactivate_worker(state, worker_id, error, logs) - - {:ok, {:error, _reason}} -> - record_poll_failure(state, worker_id) - - :error -> - record_poll_failure(state, worker_id) - end - end) - |> update_worker(worker_id, &%{&1 | last_poll_at: now, polling: true}) + case worker_launcher(state, worker) do + {:ok, launcher} -> + state + |> Fleet.call_launcher(launcher, :poll, [worker.data, launcher], fn state, result -> + state = update_worker(state, worker_id, &%{&1 | polling: false}) + + case result do + {:ok, {:ok, true}} -> + clear_poll_failures(state, worker_id) + + {:ok, {:ok, false, error, logs}} -> + # The launcher knows the worker has gone, and this is the + # only place its exit code and log tail come from. + Fleet.deactivate_worker(state, worker_id, error, logs) + + {:ok, {:error, _reason}} -> + record_poll_failure(state, worker_id) + + :error -> + record_poll_failure(state, worker_id) + end + end) + |> update_worker(worker_id, &%{&1 | last_poll_at: now, polling: true}) + + # Nothing to ask the launcher with: a poll that couldn't be + # answered, and tolerated the same way. + {:error, _reason} -> + state + |> update_worker(worker_id, &%{&1 | last_poll_at: now}) + |> record_poll_failure(worker_id) + end end) # A worker that connected but never said what it can run is broken @@ -700,7 +651,6 @@ defmodule Coflux.Orchestration.Server.Scheduler do |> Enum.filter(fn {_worker_id, worker} -> stop_due?(state, worker, now) end) |> Enum.reduce(state, fn {worker_id, worker}, state -> {:ok, worker_stop_id, stopping_at} = Workers.create_worker_stop(state.db, worker_id) - {:ok, launcher} = Workspaces.get_launcher_for_pool(state.db, worker.pool_id) state = state @@ -712,27 +662,34 @@ defmodule Coflux.Orchestration.Server.Scheduler do stopping_at: stopping_at }) - Fleet.call_launcher(state, launcher, :stop, [worker.data], fn state, result -> - case result do - {:ok, :ok} -> - {:ok, stopped_at} = - Workers.create_worker_stop_result(state.db, worker_stop_id, nil) - - Effects.emit(state, %WorkerStopResult{ - workspace: State.workspace_external_id(state, worker.workspace_id), - pool: worker.pool_name, - worker: worker.external_id, - stopped_at: stopped_at, - error: nil - }) - - {:ok, {:error, reason}} -> - record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) - - :error -> - record_stop_failure(state, worker_id, worker, worker_stop_id, "stop_crashed") - end - end) + case worker_launcher(state, worker) do + {:ok, launcher} -> + Fleet.call_launcher(state, launcher, :stop, [worker.data, launcher], fn state, + result -> + case result do + {:ok, :ok} -> + {:ok, stopped_at} = + Workers.create_worker_stop_result(state.db, worker_stop_id, nil) + + Effects.emit(state, %WorkerStopResult{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + stopped_at: stopped_at, + error: nil + }) + + {:ok, {:error, reason}} -> + record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) + + :error -> + record_stop_failure(state, worker_id, worker, worker_stop_id, "stop_crashed") + end + end) + + {:error, reason} -> + record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) + end end) # While any worker exists there are deadlines to sweep for - polls, @@ -833,6 +790,150 @@ defmodule Coflux.Orchestration.Server.Scheduler do end end + # Starts a worker's launch, with the pool's secrets resolved for the + # launcher to use. A secret that doesn't resolve fails the launch the + # way the launcher failing would, so the worker records what went wrong. + defp start_launch( + state, + workspace_id, + pool_id, + pool_name, + pool, + worker_id, + worker_external_id, + token + ) do + case resolve_launcher(state, workspace_id, pool.launcher) do + {:ok, launcher} -> + Fleet.call_launcher( + state, + launcher, + :launch, + [ + Fleet.build_launcher_env(state, workspace_id, token, launcher), + pool.modules, + launcher, + %{pool_name: pool_name} + ], + launch_callback( + workspace_id, + pool_id, + pool_name, + worker_id, + worker_external_id, + launcher + ) + ) + + {:error, error, detail} -> + callback = + launch_callback(workspace_id, pool_id, pool_name, worker_id, worker_external_id, nil) + + callback.(state, {:ok, {:error, error, detail}}) + end + end + + defp launch_callback(workspace_id, pool_id, pool_name, worker_id, worker_external_id, launcher) do + fn state, result -> + # A launcher can say more than a code about why a + # launch failed - an API's own message, typically - + # and that goes where a log tail would. + {data, error, detail} = + case result do + {:ok, {:ok, data}} -> {data, nil, nil} + {:ok, {:error, error}} -> {nil, error, nil} + {:ok, {:error, error, detail}} -> {nil, error, detail} + :error -> {nil, "launch_crashed", nil} + end + + {:ok, started_at} = + Workers.create_worker_launch_result(state.db, worker_id, data, error) + + state = + Effects.emit(state, %WorkerLaunchResult{ + workspace: State.workspace_external_id(state, workspace_id), + pool: pool_name, + worker: worker_external_id, + started_at: started_at, + error: error + }) + + cond do + error -> + # Deactivating the worker pops it from state, and + # with it the only record that this pool was ever + # tried - so count the failure first, or the pool + # relaunches on the very next pass. + state + |> record_pool_launch_failure(pool_id) + |> Fleet.deactivate_worker(worker_id, error, detail) + + Map.has_key?(state.workers, worker_id) -> + put_in( + state, + [Access.key(:workers), worker_id, Access.key(:data)], + data + ) + + true -> + # The worker was deactivated while its launch was + # in flight. Nothing will ever connect to what was + # just started, and this result is the only thing + # that knows how to reach it, so stop it here + # rather than leaking it. + Fleet.call_launcher( + state, + launcher, + :stop, + [data, launcher], + fn state, _result -> state end + ) + end + end + end + + # The launcher config with its secrets' values, for one call and then + # forgotten. Failures are in the launcher's terms: a code for the + # worker, and what was wrong for its logs. + defp resolve_launcher(state, workspace_id, launcher) do + workspace_name = state.workspaces[workspace_id].name + + case Coflux.Admin.Secrets.resolve_launcher( + state.admin_db, + state.project_id, + workspace_name, + launcher + ) do + {:ok, launcher} -> + {:ok, launcher} + + {:error, {:secret_not_found, name}} -> + {:error, "launch_secret_missing", "secret not found: #{name}"} + + {:error, {:secret_invalid, name}} -> + {:error, "launch_secret_invalid", "secret can't be used: #{name}"} + + {:error, :no_secret} -> + {:error, "launch_secret_missing", "secrets need COFLUX_SECRET to be configured"} + end + end + + # A worker's launcher config as its pool has it now, secrets resolved. + defp worker_launcher(state, worker) do + case Workspaces.get_launcher_for_pool(state.db, worker.pool_id) do + {:ok, nil} -> + {:error, :no_launcher} + + {:ok, launcher} -> + Coflux.Admin.Secrets.resolve_launcher( + state.admin_db, + state.project_id, + state.workspaces[worker.workspace_id].name, + launcher + ) + end + end + defp poll_due?(state, worker, now) do cond do is_nil(worker.data) -> @@ -939,6 +1040,8 @@ defmodule Coflux.Orchestration.Server.Scheduler do defp earliest(a, nil), do: a defp earliest(a, b), do: min(a, b) + defp to_error({:secret_not_found, name}), do: "secret_missing:#{name}" + defp to_error({:secret_invalid, name}), do: "secret_invalid:#{name}" defp to_error(reason) when is_binary(reason), do: reason defp to_error(reason) when is_atom(reason), do: Atom.to_string(reason) defp to_error(reason), do: inspect(reason) diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index f8f76bfa..a8d1cbe9 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -51,6 +51,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do StreamClosed, StreamItemAppended, StreamRegistered, + SecretSet, TokenCreated, WorkerCreated, WorkerDeactivated, @@ -207,6 +208,22 @@ defmodule Coflux.Orchestration.Server.Snapshots do end end + def load(state, :secrets, _opts) do + {:ok, secrets} = Coflux.Admin.Secrets.list(state.admin_db) + + {:ok, + Enum.map(secrets, fn secret -> + %SecretSet{ + scope: secret.scope, + name: secret.name, + version: secret.version, + created_at: secret.created_at, + updated_at: secret.updated_at, + updated_by: secret.updated_by + } + end)} + end + # Revoked tokens are absent: created-then-revoked folds to absence. def load(state, :tokens, _opts) do {:ok, tokens} = Coflux.Admin.Tokens.list_tokens(state.admin_db) diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index 1ee5fb39..8581902e 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -570,14 +570,15 @@ defmodule Coflux.Orchestration.Workspaces do {:ok, launcher_patch} -> existing_launcher = pool[:launcher] || %{} - # Apply each field from the patch, with special handling for :env merging + # Apply each field from the patch. The environment maps merge by + # key, so a patch can set or unset one variable. new_launcher = Enum.reduce(launcher_patch, existing_launcher, fn - {:env, :unset}, acc -> - Map.delete(acc, :env) + {key, :unset}, acc when key in [:env, :env_secrets] -> + Map.delete(acc, key) - {:env, env_patch}, acc when is_map(env_patch) -> - existing_env = Map.get(acc, :env, %{}) + {key, env_patch}, acc when key in [:env, :env_secrets] and is_map(env_patch) -> + existing_env = Map.get(acc, key, %{}) merged_env = Enum.reduce(env_patch, existing_env, fn @@ -586,9 +587,9 @@ defmodule Coflux.Orchestration.Workspaces do end) if merged_env == %{} do - Map.delete(acc, :env) + Map.delete(acc, key) else - Map.put(acc, :env, merged_env) + Map.put(acc, key, merged_env) end {key, :unset}, acc -> @@ -838,13 +839,15 @@ defmodule Coflux.Orchestration.Workspaces do # them back so that ProcessLauncher / build_launcher_env can use them # directly with String.to_charlist/1. config = - case Map.get(config, :env) do - env when is_map(env) and map_size(env) > 0 -> - Map.put(config, :env, Map.new(env, fn {k, v} -> {to_string(k), v} end)) + Enum.reduce([:env, :env_secrets], config, fn key, config -> + case Map.get(config, key) do + map when is_map(map) and map_size(map) > 0 -> + Map.put(config, key, Map.new(map, fn {k, v} -> {to_string(k), v} end)) - _ -> - config - end + _ -> + config + end + end) Map.put(config, :type, type) end diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index a88f68c5..c61e2794 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -60,15 +60,9 @@ defmodule Coflux.Topics.Pool do :process -> %{type: "process", directory: launcher.directory} - # `token` is deliberately absent: this shape is delivered to every - # subscriber of the topic, and the launcher's credentials are not - # part of what a pool looks like. `caCert` is a path on the - # server's host rather than a credential, so it stays. Everything - # else a pool is configured with belongs here too, or `pools get` - # shows less than `pools export` does. - # As above: the secret access key and session token are credentials, - # so they stay out. The key ID says which credentials without being - # one, and is how a pool's access is recognised, so it stays. + # Credentials are named, never held, so everything a pool is + # configured with belongs here: `pools get` shows exactly what + # `pools export` does. :ecs -> %{ type: "ecs", @@ -83,7 +77,7 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:securityGroups, Map.get(launcher, :security_groups)) |> maybe_put(:assignPublicIp, Map.get(launcher, :assign_public_ip)) |> maybe_put(:platformVersion, Map.get(launcher, :platform_version)) - |> maybe_put(:accessKeyId, Map.get(launcher, :access_key_id)) + |> maybe_put(:credentialsSecret, Map.get(launcher, :credentials_secret)) |> maybe_put(:endpoint, Map.get(launcher, :endpoint)) :kubernetes -> @@ -91,6 +85,7 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:namespace, Map.get(launcher, :namespace)) |> maybe_put(:apiServer, Map.get(launcher, :api_server)) |> maybe_put(:serviceAccount, Map.get(launcher, :service_account)) + |> maybe_put(:tokenSecret, Map.get(launcher, :token_secret)) |> maybe_put(:caCert, Map.get(launcher, :ca_cert)) |> maybe_put(:insecure, Map.get(launcher, :insecure)) |> maybe_put(:imagePullPolicy, Map.get(launcher, :image_pull_policy)) @@ -112,6 +107,7 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:adapter, Map.get(launcher, :adapter)) |> maybe_put(:concurrency, Map.get(launcher, :concurrency)) |> maybe_put(:env, Map.get(launcher, :env)) + |> maybe_put(:envSecrets, Map.get(launcher, :env_secrets)) end defp maybe_put(map, _key, nil), do: map diff --git a/server/lib/coflux/topics/secrets.ex b/server/lib/coflux/topics/secrets.ex new file mode 100644 index 00000000..49bde170 --- /dev/null +++ b/server/lib/coflux/topics/secrets.ex @@ -0,0 +1,94 @@ +defmodule Coflux.Topics.Secrets do + @moduledoc """ + The project's secrets - what exists, in which scope, and when it last + changed. Never a value. + """ + + use Topical.Topic, route: ["secrets"] + + alias Coflux.Orchestration + alias Coflux.Topics.Diff + alias Coflux.Topics.Secrets.Model + + def connect(params, context) do + {:ok, Map.put(params, :project, context.project)} + end + + def init(params) do + project_id = Map.fetch!(params, :project) + {:ok, events, ref} = Orchestration.subscribe(project_id, :secrets, self()) + {model, _dirty} = Model.fold(Model.new(), events) + {:ok, Topic.new(Model.project(model), %{model: model, ref: ref})} + end + + def handle_info({:topic, _ref, events}, topic) do + {model, dirty} = Model.fold(topic.state.model, events) + + topic = + Enum.reduce(dirty, topic, fn key, topic -> + Diff.apply(topic, [key], Map.get(topic.value, key), Model.project_entry(model, key)) + end) + + {:ok, %{topic | state: %{topic.state | model: model}}} + end +end + +defmodule Coflux.Topics.Secrets.Model do + @moduledoc false + + import Kernel, except: [apply: 2] + import Coflux.TopicUtils, only: [build_principal: 1] + + alias Coflux.Events.{SecretDeleted, SecretSet} + + def new, do: %{} + + def fold(model, events) do + Enum.reduce(events, {model, MapSet.new()}, fn event, {model, dirty} -> + {model, keys} = apply(model, event) + {model, Enum.into(keys, dirty)} + end) + end + + def apply(model, %SecretSet{} = e) do + key = key(e.scope, e.name) + + secret = %{ + scope: e.scope, + name: e.name, + version: e.version, + created_at: e.created_at, + updated_at: e.updated_at, + updated_by: e.updated_by + } + + {Map.put(model, key, secret), [key]} + end + + def apply(model, %SecretDeleted{} = e) do + key = key(e.scope, e.name) + {Map.delete(model, key), [key]} + end + + # A name is unique within a scope, and a scope can be empty. + defp key(scope, name), do: "#{name}@#{scope}" + + def project(model), do: Map.new(model, fn {key, _} -> {key, project_entry(model, key)} end) + + def project_entry(model, key) do + case Map.fetch(model, key) do + {:ok, secret} -> + %{ + scope: secret.scope, + name: secret.name, + version: secret.version, + createdAt: secret.created_at, + updatedAt: secret.updated_at, + updatedBy: build_principal(secret.updated_by) + } + + :error -> + nil + end + end +end diff --git a/server/priv/migrations/admin/2.sql b/server/priv/migrations/admin/2.sql new file mode 100644 index 00000000..814c546b --- /dev/null +++ b/server/priv/migrations/admin/2.sql @@ -0,0 +1,24 @@ +-- Secrets: values a pool needs that must never appear in its +-- configuration. Encrypted with a key derived from COFLUX_SECRET and bound +-- to their row (project, scope, name, version), so a ciphertext moved +-- elsewhere won't decrypt. One current value per scope and name: setting +-- it again replaces the value and bumps the version, deleting it removes +-- it, and nothing older is kept. +-- +-- `scope` is a workspace name, or a prefix of one ('development' covers +-- 'development/joe'); '' is the whole project. +CREATE TABLE secrets ( + id INTEGER PRIMARY KEY, + scope TEXT NOT NULL, + name TEXT NOT NULL, + version INTEGER NOT NULL, + key_id TEXT NOT NULL, + nonce BLOB NOT NULL, + ciphertext BLOB NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + updated_by_type TEXT, -- 'user' or 'token' + updated_by_external_id TEXT, + UNIQUE (scope, name), + CHECK ((updated_by_type IS NULL) = (updated_by_external_id IS NULL)) +) STRICT; diff --git a/server/test/coflux/admin_secrets_test.exs b/server/test/coflux/admin_secrets_test.exs new file mode 100644 index 00000000..6b0ca716 --- /dev/null +++ b/server/test/coflux/admin_secrets_test.exs @@ -0,0 +1,134 @@ +defmodule Coflux.AdminSecretsTest do + use ExUnit.Case, async: true + + alias Coflux.Admin.Secrets + alias Coflux.Store + alias Coflux.Store.Migrations + alias Exqlite.Sqlite3 + + @project "proj" + @by %{type: "user", external_id: "user-1"} + + setup_all do + previous = :persistent_term.get(:coflux_secret, nil) + :persistent_term.put(:coflux_secret, "test-secret") + on_exit(fn -> :persistent_term.put(:coflux_secret, previous) end) + :ok + end + + setup do + {:ok, db} = Sqlite3.open(":memory:") + :ok = Migrations.run(db, "admin") + {:ok, db: db} + end + + test "a value round-trips, and setting it again replaces it and bumps the version", %{db: db} do + assert {:ok, %{version: 1}} = Secrets.set(db, @project, "", "api-key", "first", @by) + assert {:ok, "first"} = Secrets.resolve(db, @project, "development", "api-key") + + assert {:ok, %{version: 2, updated_by: @by}} = + Secrets.set(db, @project, "", "api-key", "second", @by) + + assert {:ok, "second"} = Secrets.resolve(db, @project, "development", "api-key") + + assert {:ok, [%{name: "api-key", scope: "", version: 2}]} = Secrets.list(db) + assert :ok = Secrets.delete(db, "", "api-key") + assert {:error, :not_found} = Secrets.delete(db, "", "api-key") + assert {:error, :not_found} = Secrets.resolve(db, @project, "development", "api-key") + end + + test "the nearest scope wins, and production's secrets don't reach development", %{db: db} do + {:ok, _} = Secrets.set(db, @project, "", "key", "project-wide", nil) + {:ok, _} = Secrets.set(db, @project, "development", "key", "for-development", nil) + {:ok, _} = Secrets.set(db, @project, "development/joe", "key", "for-joe", nil) + {:ok, _} = Secrets.set(db, @project, "production", "key", "for-production", nil) + + assert {:ok, "for-joe"} = Secrets.resolve(db, @project, "development/joe", "key") + assert {:ok, "for-joe"} = Secrets.resolve(db, @project, "development/joe/feature", "key") + assert {:ok, "for-development"} = Secrets.resolve(db, @project, "development/sam", "key") + assert {:ok, "project-wide"} = Secrets.resolve(db, @project, "staging", "key") + assert {:ok, "for-production"} = Secrets.resolve(db, @project, "production", "key") + + # A prefix is a path prefix, not a string prefix. + assert {:ok, "project-wide"} = Secrets.resolve(db, @project, "developments", "key") + + {:ok, _} = Secrets.set(db, @project, "production", "prod-only", "x", nil) + assert {:error, :not_found} = Secrets.resolve(db, @project, "development/joe", "prod-only") + end + + test "a ciphertext is bound to its row", %{db: db} do + {:ok, _} = Secrets.set(db, @project, "", "a", "value-a", nil) + {:ok, _} = Secrets.set(db, @project, "", "b", "value-b", nil) + + {:ok, _} = + Store.query( + db, + """ + UPDATE secrets SET (nonce, ciphertext) = + (SELECT nonce, ciphertext FROM secrets WHERE name = 'a') + WHERE name = 'b' + """, + {} + ) + + assert {:ok, "value-a"} = Secrets.resolve(db, @project, "x", "a") + assert {:error, :undecryptable} = Secrets.resolve(db, @project, "x", "b") + assert {:error, :undecryptable} = Secrets.resolve(db, "other-project", "x", "a") + end + + test "a launcher config is given the values it names", %{db: db} do + {:ok, _} = Secrets.set(db, @project, "", "k8s-token", "bearer-xyz", nil) + {:ok, _} = Secrets.set(db, @project, "", "openai", "sk-123", nil) + + {:ok, _} = + Secrets.set( + db, + @project, + "", + "aws", + ~s({"Version": 1, "AccessKeyId": "AKIA1", "SecretAccessKey": "s3cr3t", "SessionToken": "tok"}), + nil + ) + + launcher = %{ + type: :kubernetes, + token_secret: "k8s-token", + env: %{"PLAIN" => "1"}, + env_secrets: %{"OPENAI_API_KEY" => "openai"} + } + + assert Secrets.references(launcher) == ["k8s-token", "openai"] + assert :ok = Secrets.check_references(db, "dev", launcher) + + assert {:ok, resolved} = Secrets.resolve_launcher(db, @project, "dev", launcher) + assert resolved.token == "bearer-xyz" + assert resolved.env == %{"PLAIN" => "1", "OPENAI_API_KEY" => "sk-123"} + + ecs = %{type: :ecs, credentials_secret: "aws"} + assert {:ok, resolved} = Secrets.resolve_launcher(db, @project, "dev", ecs) + assert resolved.access_key_id == "AKIA1" + assert resolved.secret_access_key == "s3cr3t" + assert resolved.session_token == "tok" + + missing = %{type: :kubernetes, token_secret: "nope", env_secrets: %{"X" => "openai"}} + assert {:error, {:secrets_not_found, ["nope"]}} = Secrets.check_references(db, "dev", missing) + + assert {:error, {:secret_not_found, "nope"}} = + Secrets.resolve_launcher(db, @project, "dev", missing) + + {:ok, _} = Secrets.set(db, @project, "", "not-json", "plain text", nil) + + assert {:error, {:secret_invalid, "not-json"}} = + Secrets.resolve_launcher(db, @project, "dev", %{credentials_secret: "not-json"}) + end + + test "without a server secret nothing can be stored or read", %{db: db} do + :persistent_term.put(:coflux_secret, nil) + + try do + assert {:error, :no_secret} = Secrets.set(db, @project, "", "x", "v", nil) + after + :persistent_term.put(:coflux_secret, "test-secret") + end + end +end diff --git a/tests/support/cli.py b/tests/support/cli.py index ecfe495c..04347063 100644 --- a/tests/support/cli.py +++ b/tests/support/cli.py @@ -19,7 +19,13 @@ def _build_env(env_vars=None): def _coflux( - *args, host=None, workspace="default", output="json", env_vars=None, timeout=30 + *args, + host=None, + workspace="default", + output="json", + env_vars=None, + timeout=30, + input=None, ): cmd = [_COFLUX_BIN] if host: @@ -36,6 +42,7 @@ def _coflux( env=_build_env(env_vars), check=True, timeout=timeout, + input=input, ) @@ -381,14 +388,35 @@ def pools_disable(name, host=None, workspace="default"): _coflux("pools", "disable", name, host=host, workspace=workspace, output=None) -def pools_export(include_secrets=False, host=None, workspace="default"): - args = ["pools", "export"] - if include_secrets: - args.append("--include-secrets") - result = _coflux(*args, host=host, workspace=workspace, output=None) +def pools_export(host=None, workspace="default"): + result = _coflux("pools", "export", host=host, workspace=workspace, output=None) return result.stdout +def _secret_scope_args(scope, global_): + if global_: + return ["--global"] + if scope is not None: + return ["--scope", scope] + return [] + + +def secrets_set(name, value, scope=None, global_=False, host=None, workspace="default"): + """Set a secret, with the value on stdin as a user would give it.""" + args = ["secrets", "set", name, *_secret_scope_args(scope, global_)] + _coflux(*args, host=host, workspace=workspace, output=None, input=value) + + +def secrets_list(host=None, workspace="default"): + result = _coflux("secrets", "list", host=host, workspace=workspace) + return json.loads(result.stdout) + + +def secrets_delete(name, scope=None, global_=False, host=None, workspace="default"): + args = ["secrets", "delete", name, *_secret_scope_args(scope, global_)] + _coflux(*args, host=host, workspace=workspace, output=None) + + def pools_import(path, host=None, workspace="default"): _coflux( "pools", diff --git a/tests/support/server.py b/tests/support/server.py index 7dd8ac67..553ea4a1 100644 --- a/tests/support/server.py +++ b/tests/support/server.py @@ -87,6 +87,8 @@ def _start_local(self, timeout): # is exercised against a stand-in for the API (support.ecs). "COFLUX_LAUNCHER_TYPES": "process,docker,kubernetes,ecs", "COFLUX_SUPER_TOKEN_HASH": hashlib.sha256(SUPER_TOKEN.encode()).hexdigest(), + # Signs service tokens and encrypts secrets. + "COFLUX_SECRET": "test-secret", "COFLUX_CLI_PATH": cli_path, **self._extra_env, } @@ -118,6 +120,8 @@ def _start_docker(self, image, timeout): "COFLUX_REQUIRE_AUTH=false", "-e", f"COFLUX_SUPER_TOKEN_HASH={hashlib.sha256(SUPER_TOKEN.encode()).hexdigest()}", + "-e", + "COFLUX_SECRET=test-secret", image, ], check=True, diff --git a/tests/test_pools.py b/tests/test_pools.py index c5399ea0..def1314e 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -308,7 +308,7 @@ def test_idle_timeout_is_a_pool_field(self, pool_env, tmp_path): assert "idleTimeout" not in pool["launcher"] exported = cli.pools_export(host=host) - assert "idle_timeout = 300" in exported + assert "idle_timeout = 300\n" in exported cli.pools_update("idle-pool", idle_timeout=0, host=host) assert cli.pools_get("idle-pool", host=host)["idleTimeout"] == 0 @@ -603,7 +603,11 @@ def test_wildcard_modules_are_rejected(self, pool_env): class TestPoolSecrets: - def _create_kubernetes_pool(self, host, name="k8s-pool"): + """Pools name secrets; launchers get their values, and nothing else does.""" + + def _kubernetes_pool( + self, host, name="k8s-pool", token_secret="k8s-token", **kwargs + ): cli._coflux( "pools", "create", @@ -613,56 +617,150 @@ def _create_kubernetes_pool(self, host, name="k8s-pool"): "--set", "image=myorg/worker:latest", "--set", - "token=super-secret-token", + f"tokenSecret={token_secret}", "--set", "apiServer=https://k8s.example.com", "--modules", "test", host=host, output=None, + **kwargs, ) - def test_export_redacts_secrets_by_default(self, pool_env): - """An export doesn't put the cluster token on disk unasked.""" + def test_secret_env_reaches_worker(self, pool_env): + """A secret named in envSecrets is in the worker's environment, and + its value shows up nowhere a pool is described.""" host = pool_env["host"] - self._create_kubernetes_pool(host) + worker_dir = pool_env["worker_dir"] + executor = pool_env["executor"] + marker = str(worker_dir / "secret_marker.txt") + wrapper_script = str(worker_dir / "secret_wrapper.py") + manifest_path = pool_env["manifest_path"] + socket_path = pool_env["socket_path"] - exported = cli.pools_export(host=host) - assert "super-secret-token" not in exported - assert "" in exported - # Everything that isn't a secret is still there. - assert "https://k8s.example.com" in exported + with open(wrapper_script, "w") as f: + f.write( + "import os, sys, subprocess\n" + f"with open({marker!r}, 'w') as f:\n" + " f.write(os.environ.get('TEST_SECRET', ''))\n" + "result = subprocess.run(\n" + f" ['python3', {ADAPTER_SCRIPT!r}] + sys.argv[1:],\n" + " stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr\n" + ")\n" + "sys.exit(result.returncode)\n" + ) + + with open(manifest_path, "w") as f: + json.dump(manifest([workflow("test", "check_env")]), f) + + adapter = [ + "python3", + wrapper_script, + "--manifest", + manifest_path, + "--socket", + socket_path, + ] + + # Set for the current workspace, which is where the pool is. + cli.secrets_set("api-key", "s3cr3t-value", host=host) - def test_export_with_secrets_round_trips(self, pool_env, tmp_path): - """--include-secrets gives a config that imports back unchanged.""" + cli.pools_create( + "secret-pool", + type="process", + modules=["test"], + process_dir=str(worker_dir), + adapter=adapter, + host=host, + ) + cli._coflux( + "pools", + "update", + "secret-pool", + "--set", + "envSecrets.TEST_SECRET=api-key", + host=host, + output=None, + ) + cli.manifests_register("test", adapter=",".join(adapter), host=host) + + cli.submit("test/check_env", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + + with open(marker) as f: + assert f.read() == "s3cr3t-value" + + pool = cli.pools_get("secret-pool", host=host) + assert pool["launcher"]["envSecrets"] == {"TEST_SECRET": "api-key"} + assert "s3cr3t-value" not in json.dumps(pool) + assert "s3cr3t-value" not in cli.pools_export(host=host) + + def test_missing_secret_is_refused(self, pool_env): + """A pool naming a secret that doesn't exist for its workspace is + refused when it's created or updated, naming the secret.""" host = pool_env["host"] - self._create_kubernetes_pool(host) - exported = cli.pools_export(include_secrets=True, host=host) - assert "super-secret-token" in exported + with pytest.raises(subprocess.CalledProcessError) as exc_info: + self._kubernetes_pool(host, token_secret="nope") + assert "secrets_not_found" in exc_info.value.stderr + assert "nope" in exc_info.value.stderr - path = tmp_path / "pools.toml" - path.write_text(exported) - cli.pools_import(path, host=host) + cli.secrets_set("k8s-token", "bearer", host=host) + self._kubernetes_pool(host) - again = cli.pools_export(include_secrets=True, host=host) - assert "super-secret-token" in again + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli._coflux( + "pools", + "update", + "k8s-pool", + "--set", + "envSecrets.X=missing", + host=host, + output=None, + ) + assert "secrets_not_found" in exc_info.value.stderr - def test_importing_a_redacted_export_is_refused(self, pool_env, tmp_path): - """A redacted export can't silently clear the secrets it omits.""" + def test_scope_follows_workspace_names(self, pool_env): + """A secret for 'development' serves 'development/joe' and not + 'production', whatever the workspaces inherit from.""" host = pool_env["host"] - self._create_kubernetes_pool(host) + cli.secrets_set("k8s-token", "bearer", scope="development", host=host) - path = tmp_path / "pools.toml" - path.write_text(cli.pools_export(host=host)) + self._kubernetes_pool(host, workspace="development/joe") with pytest.raises(subprocess.CalledProcessError) as exc_info: - cli.pools_import(path, host=host) - assert "k8s-pool" in exc_info.value.stderr - assert "--include-secrets" in exc_info.value.stderr + self._kubernetes_pool(host, workspace="production") + assert "secrets_not_found" in exc_info.value.stderr - # And the real token is untouched. - assert "super-secret-token" in cli.pools_export(include_secrets=True, host=host) + def test_export_names_secrets(self, pool_env, tmp_path): + """An export carries the names, never the values, and imports back + as long as the secrets exist.""" + host = pool_env["host"] + cli.secrets_set("k8s-token", "super-secret-token", host=host) + self._kubernetes_pool(host) + cli._coflux( + "pools", + "update", + "k8s-pool", + "--set", + "envSecrets.API_KEY=k8s-token", + host=host, + output=None, + ) + + exported = cli.pools_export(host=host) + assert 'token_secret = "k8s-token"' in exported + assert "env_secrets = {" in exported + assert 'API_KEY = "k8s-token"' in exported + assert "super-secret-token" not in exported + + path = tmp_path / "pools.toml" + path.write_text(exported) + cli.pools_import(path, host=host) + + launcher = cli.pools_get("k8s-pool", host=host)["launcher"] + assert launcher["tokenSecret"] == "k8s-token" + assert launcher["envSecrets"] == {"API_KEY": "k8s-token"} # --------------------------------------------------------------------------- @@ -707,14 +805,21 @@ def _setup_ecs_pool(ecs_env, targets, modules=None, pool_name="ecs-pool", sets=( "taskDefinition=worker-task", "region=us-east-1", f"endpoint={fake.endpoint}", - "accessKeyId=AKIATEST", - "secretAccessKey=test-secret-key", + "credentialsSecret=aws-test", 'subnets=["subnet-1", "subnet-2"]', "securityGroups=sg-1", "assignPublicIp=true", f"adapter={json.dumps(adapter)}", *sets, ] + # In the shape `aws configure export-credentials` produces. + cli.secrets_set( + "aws-test", + json.dumps({"AccessKeyId": "AKIATEST", "SecretAccessKey": "test-secret-key"}), + global_=True, + host=host, + ) + args = ["pools", "create", pool_name, "--type", "ecs"] for field in fields: args.extend(["--set", field]) @@ -915,9 +1020,7 @@ def test_single_ids_are_accepted_for_lists(self, ecs_env): assert launcher["subnets"] == ["subnet-9"] assert launcher["securityGroups"] == ["sg-1"] assert launcher["assignPublicIp"] is True - # The key ID identifies the credentials; the secret stays out. - assert launcher["accessKeyId"] == "AKIATEST" - assert "secretAccessKey" not in launcher + assert launcher["credentialsSecret"] == "aws-test" cli._coflux( "pools", @@ -932,32 +1035,20 @@ def test_single_ids_are_accepted_for_lists(self, ecs_env): "subnet-10" ] - def test_export_redacts_credentials(self, ecs_env, tmp_path): - """The secret key and session token are secrets; the key ID isn't.""" + def test_export_names_the_credentials_secret(self, ecs_env, tmp_path): + """An export carries the secret's name, never its value, and + imports back while the secret exists.""" host = ecs_env["host"] - _setup_ecs_pool( - ecs_env, - [workflow("test", "greet")], - sets=["sessionToken=test-session-token"], - ) + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) exported = cli.pools_export(host=host) - assert "test-secret-key" not in exported - assert "test-session-token" not in exported - assert 'secret_access_key = ""' in exported - assert 'session_token = ""' in exported - assert 'access_key_id = "AKIATEST"' in exported + assert 'credentials_secret = "aws-test"' in exported assert 'task_definition = "worker-task"' in exported + assert "test-secret-key" not in exported + assert "AKIATEST" not in exported path = tmp_path / "pools.toml" path.write_text(exported) - with pytest.raises(subprocess.CalledProcessError) as exc_info: - cli.pools_import(path, host=host) - assert "ecs-pool" in exc_info.value.stderr - - with_secrets = cli.pools_export(include_secrets=True, host=host) - assert "test-secret-key" in with_secrets - assert "test-session-token" in with_secrets - path.write_text(with_secrets) cli.pools_import(path, host=host) - assert "test-secret-key" in cli.pools_export(include_secrets=True, host=host) + launcher = cli.pools_get("ecs-pool", host=host)["launcher"] + assert launcher["credentialsSecret"] == "aws-test" diff --git a/tests/test_secrets.py b/tests/test_secrets.py new file mode 100644 index 00000000..8741a155 --- /dev/null +++ b/tests/test_secrets.py @@ -0,0 +1,60 @@ +"""Secrets: set, listed and deleted by name, with values never shown.""" + +import json +import subprocess + +import pytest +from support import cli + + +class TestSecrets: + def test_set_list_delete(self, server, project_id): + host = f"{project_id}.localhost:{server.port}" + + # The default scope is the current workspace. + cli.secrets_set("api-key", "first", host=host) + [secret] = cli.secrets_list(host=host) + assert secret["name"] == "api-key" + assert secret["scope"] == "default" + assert secret["version"] == 1 + + # Setting it again replaces the value and bumps the version. + cli.secrets_set("api-key", "second\n", host=host) + [secret] = cli.secrets_list(host=host) + assert secret["version"] == 2 + + cli.secrets_set("shared", "x", global_=True, host=host) + cli.secrets_set("shared", "y", scope="development", host=host) + listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} + assert listed == { + ("default", "api-key"), + ("", "shared"), + ("development", "shared"), + } + + # Values are nowhere in the listing. + assert "first" not in json.dumps(cli.secrets_list(host=host)) + assert "second" not in json.dumps(cli.secrets_list(host=host)) + + cli.secrets_delete("shared", global_=True, host=host) + cli.secrets_delete("api-key", host=host) + listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} + assert listed == {("development", "shared")} + + with pytest.raises(subprocess.CalledProcessError): + cli.secrets_delete("api-key", host=host) + + def test_names_and_scopes_are_validated(self, server, project_id): + host = f"{project_id}.localhost:{server.port}" + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.secrets_set("not valid!", "v", host=host) + assert "bad_request" in exc_info.value.stderr + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.secrets_set("ok", "v", scope="trailing/", host=host) + assert "bad_request" in exc_info.value.stderr + + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli.secrets_set("ok", "", host=host) + assert "no value given" in exc_info.value.stderr diff --git a/tests/test_tokens.py b/tests/test_tokens.py index dd09be32..1f44415e 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -87,6 +87,30 @@ def test_token_outlives_rotation_and_restart(self, token_server): token_server.restart(timeout=30) assert _discover(port, project_id, created["token"]) == (200, ["*"]) + def test_setting_a_secret_takes_operator_access_to_its_scope(self, token_server): + """A token for 'development/*' can set a secret for a development + workspace, but not for the project or for production.""" + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + restricted = _create(port, project_id, name="dev", workspaces=["development/*"]) + + def set_secret(scope): + try: + api_post( + port, + project_id, + "set_secret", + token=restricted["token"], + body={"name": "key", "scope": scope, "value": "v"}, + ) + return 200 + except urllib.error.HTTPError as e: + return e.code + + assert set_secret("development/joe") == 200 + assert set_secret("") == 403 + assert set_secret("production") == 403 + def test_only_the_creator_or_full_access_can_revoke(self, token_server): """Who created a token is kept with it, and resolved to a principal in whichever epoch asks - here, after a rotation.""" From c14cc6156a144b83450169886cc251a0f8c3dd0b Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 08:10:18 +0100 Subject: [PATCH 07/21] Fix workspace/scope matching --- cli/cmd/coflux/tokens.go | 8 +- docs/docs/authentication.md | 2 + server/CHANGELOG.md | 4 +- server/lib/coflux/admin/secrets.ex | 9 +-- server/lib/coflux/auth.ex | 9 ++- server/lib/coflux/handlers/api.ex | 50 ++++-------- server/lib/coflux/orchestration/server.ex | 10 +-- .../orchestration/server/permissions.ex | 17 +---- server/lib/coflux/scopes.ex | 68 +++++++++++++++++ server/test/coflux/scopes_test.exs | 76 +++++++++++++++++++ tests/test_tokens.py | 73 ++++++++++++++++-- 11 files changed, 249 insertions(+), 77 deletions(-) create mode 100644 server/lib/coflux/scopes.ex create mode 100644 server/test/coflux/scopes_test.exs diff --git a/cli/cmd/coflux/tokens.go b/cli/cmd/coflux/tokens.go index 3857d84c..e0add58d 100644 --- a/cli/cmd/coflux/tokens.go +++ b/cli/cmd/coflux/tokens.go @@ -97,9 +97,11 @@ var tokensCreateCmd = &cobra.Command{ The token value is displayed only once. Make sure to copy it. -Use --workspaces to restrict the token to specific workspaces. Patterns can -include wildcards (e.g., 'development/*' matches all workspaces starting with -'development/'). If omitted, the token has access to all workspaces.`, +Use --workspaces to restrict the token to specific workspaces. A pattern grants +a workspace and everything under it, so 'development' and 'development/*' both +cover 'development' and 'development/joe'. Use '*' for every workspace. A token +can't be given broader access than the one creating it. If omitted, the token +inherits the access of the token creating it.`, RunE: runTokensCreate, } diff --git a/docs/docs/authentication.md b/docs/docs/authentication.md index 53453121..92173b0d 100644 --- a/docs/docs/authentication.md +++ b/docs/docs/authentication.md @@ -42,6 +42,8 @@ Service tokens can be created and scoped to specific workspaces, making them sui coflux tokens create --name "CI" --workspaces "production/*" ``` +A pattern grants a workspace and everything under it, following the `/` in workspace names. So `production` and `production/*` both cover `production` itself and `production/eu`, but neither covers `production-2`. Use `*` for every workspace. A token can't be created with broader access than the token creating it has, and one created without `--workspaces` inherits the creating token's access. + Service tokens require `COFLUX_SECRET` to be configured on the server. This secret is used to derive per-project signing keys. To manage tokens: diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 5233e3c6..3c9e470e 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.13.0 -No changes. +Changes: + +- A workspace pattern on a token now grants the workspace it names *and everything under it*, so `development/*` covers `development` itself (as documented, but not previously implemented), and a plain `development` covers `development/joe`. Previously a plain pattern matched one workspace exactly, and `development/*` matched everything below `development` but not `development` itself. Tokens already issued widen accordingly, always to workspaces under a name the token already held. Patterns that name nothing (an empty pattern) are now rejected when creating a token. ## 0.12.0 diff --git a/server/lib/coflux/admin/secrets.ex b/server/lib/coflux/admin/secrets.ex index ed275988..6c51c7d8 100644 --- a/server/lib/coflux/admin/secrets.ex +++ b/server/lib/coflux/admin/secrets.ex @@ -9,9 +9,9 @@ defmodule Coflux.Admin.Secrets do value and bumps the version; deleting it removes it. Nothing older is kept, so a rotated or deleted value is gone from here. - A secret is scoped to a workspace name, or to a prefix of one: a secret + A secret is set for a scope, as `Coflux.Scopes` defines one: a secret for `development` applies to `development/joe`, and the nearest scope - wins. The empty scope is the whole project. This follows the naming + wins. The root scope is the whole project. This follows the naming hierarchy rather than the base-workspace chain: a workspace that inherits results from `production` doesn't inherit its secrets. @@ -178,10 +178,7 @@ defmodule Coflux.Admin.Secrets do def exists?(db, workspace_name, name), do: match?({:ok, _}, find(db, workspace_name, name)) @doc "Whether a secret in `scope` is one a workspace of this name sees." - def scope_applies?("", _workspace_name), do: true - - def scope_applies?(scope, workspace_name), - do: workspace_name == scope or String.starts_with?(workspace_name, scope <> "/") + defdelegate scope_applies?(scope, workspace_name), to: Coflux.Scopes, as: :covers? # --- Launcher configs --- diff --git a/server/lib/coflux/auth.ex b/server/lib/coflux/auth.ex index c47125c5..41a373ed 100644 --- a/server/lib/coflux/auth.ex +++ b/server/lib/coflux/auth.ex @@ -48,10 +48,11 @@ defmodule Coflux.Auth do ## Workspace Patterns - Workspace patterns control write access: - - "*" matches all workspaces - - "staging" matches exactly "staging" - - "staging/*" matches "staging", "staging/feature1", etc. + A grant is a list of workspace patterns, which control write access. + `Coflux.Scopes` defines what a pattern grants, and + `Coflux.Orchestration.Server.Permissions` applies it - the patterns are + carried through this module unread, beyond normalising a grant that + includes "*" to `:all`. """ alias Coflux.{Config, JwksStore, Orchestration} diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index d9480ed7..7423a09f 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -1,7 +1,7 @@ defmodule Coflux.Handlers.Api do import Coflux.Handlers.Utils - alias Coflux.{Auth, Config, Orchestration, MapUtils, Version} + alias Coflux.{Auth, Config, Orchestration, MapUtils, Scopes, Version} alias Coflux.Orchestration.Ids @max_parameters 20 @@ -73,46 +73,22 @@ defmodule Coflux.Handlers.Api do end end - defp workspace_matches?(_workspace, "*"), do: true - defp workspace_matches?(workspace, workspace), do: true - - defp workspace_matches?(workspace, pattern) do - if String.ends_with?(pattern, "/*") do - # "staging/*" matches "staging/foo", "staging/foo/bar", etc. (not "staging" itself) - String.starts_with?(workspace, String.slice(pattern, 0..-2//1)) - else - false - end - end - - # Check if all requested workspace patterns are covered by the caller's access. - # Returns true if the caller can grant the requested access level. + # Whether the caller can grant the requested access: every scope asked + # for has to sit within one the caller already has. Scopes are closed + # downward, so a caller's scope covering the root of a requested one + # covers all of it. defp workspaces_covered?(:all, _requested), do: true defp workspaces_covered?(_caller, nil), do: true defp workspaces_covered?(caller_patterns, requested) do - Enum.all?(requested, &pattern_covered_by?(caller_patterns, &1)) - end + caller_scopes = Enum.map(caller_patterns, &Scopes.from_pattern/1) - defp pattern_covered_by?(caller_patterns, "*") do - # Full access - only covered by explicit "*" pattern - Enum.any?(caller_patterns, &(&1 == "*")) - end - - defp pattern_covered_by?(caller_patterns, pattern) do - if String.ends_with?(pattern, "/*") do - # Wildcard pattern "X/*" - caller needs "*", same pattern, or broader wildcard - prefix = String.slice(pattern, 0..-2//1) - - Enum.any?(caller_patterns, fn cp -> - cp == "*" or cp == pattern or - (String.ends_with?(cp, "/*") and - String.starts_with?(prefix, String.slice(cp, 0..-2//1))) - end) - else - # Exact pattern - use existing workspace_matches? - Enum.any?(caller_patterns, &workspace_matches?(pattern, &1)) - end + Enum.all?(requested, fn pattern -> + case Scopes.from_pattern(pattern) do + :never -> false + scope -> Enum.any?(caller_scopes, &Scopes.covers?(&1, scope)) + end + end) end defp handle(req, "GET", ["discover"], _project_id, %{workspaces: workspaces}) do @@ -1127,7 +1103,7 @@ defmodule Coflux.Handlers.Api do # Helper functions for handle/5 clauses defp parse_workspaces(value) when is_list(value) do - if Enum.all?(value, &is_binary/1) do + if Enum.all?(value, &Scopes.valid_pattern?/1) do {:ok, value} else {:error, :invalid} diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index b42c1810..74dbed9e 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -2977,14 +2977,12 @@ defmodule Coflux.Orchestration.Server do # Private helper functions # A secret in a scope is a secret for every workspace under it, so - # setting one takes operator access to the scope itself, and the project - # scope takes access to everything. + # setting one takes operator access to the scope itself. Scopes are + # closed downward (see `Coflux.Scopes`), so a grant covering the scope's + # own name covers everything the secret can reach - asking about the + # name alone is enough. The root scope is covered only by a `*` grant. defp check_secret_scope_access(nil, _scope), do: :ok - defp check_secret_scope_access(access, "") do - if access[:workspaces] == :all, do: :ok, else: {:error, :forbidden} - end - defp check_secret_scope_access(access, scope) do if Permissions.operator?(access[:workspaces], scope), do: :ok, else: {:error, :forbidden} end diff --git a/server/lib/coflux/orchestration/server/permissions.ex b/server/lib/coflux/orchestration/server/permissions.ex index 6b3402fc..8495f061 100644 --- a/server/lib/coflux/orchestration/server/permissions.ex +++ b/server/lib/coflux/orchestration/server/permissions.ex @@ -9,7 +9,8 @@ defmodule Coflux.Orchestration.Server.Permissions do Access arrives as the caller's grant - a token's workspace patterns, or a studio session's - and `nil` means an internal caller with no - restriction. A grant matches by name pattern, and an operator grant + restriction. A pattern grants a scope of the workspace naming + hierarchy, which `Coflux.Scopes` defines, and an operator grant additionally allows the management operations (creating workspaces, editing pools, revoking tokens) that a plain workspace grant does not. @@ -19,6 +20,7 @@ defmodule Coflux.Orchestration.Server.Permissions do """ alias Coflux.Orchestration.{Sessions} + alias Coflux.Scopes def require_workspace(state, workspace_external_id, access \\ nil) do case Map.fetch(state.workspace_external_ids, workspace_external_id) do @@ -60,18 +62,7 @@ defmodule Coflux.Orchestration.Server.Permissions do def operator?(:all, _workspace), do: true def operator?(patterns, workspace) do - Enum.any?(patterns, &workspace_matches?(workspace, &1)) - end - - def workspace_matches?(_workspace, "*"), do: true - def workspace_matches?(workspace, workspace), do: true - - def workspace_matches?(workspace, pattern) do - if String.ends_with?(pattern, "/*") do - String.starts_with?(workspace, String.slice(pattern, 0..-2//1)) - else - false - end + Enum.any?(patterns, &Scopes.covers?(Scopes.from_pattern(&1), workspace)) end def check_operator_access(nil, _name), do: :ok diff --git a/server/lib/coflux/scopes.ex b/server/lib/coflux/scopes.ex new file mode 100644 index 00000000..a4933b57 --- /dev/null +++ b/server/lib/coflux/scopes.ex @@ -0,0 +1,68 @@ +defmodule Coflux.Scopes do + @moduledoc """ + The workspace naming hierarchy, and the one relation asked of it: + whether a scope covers a workspace. + + A scope is a workspace name, or a prefix of one along the `/` in the + name - `development` covers `development/joe` and `development/joe/x` - + and the root scope, `""`, covers every workspace. Names are compared a + segment at a time, so `development` doesn't cover `development-2`. + + Two things are defined in terms of this: which workspaces a secret set + for a scope applies to, and which workspaces a token's grant allows. A + grant is written as a pattern - `*` for the root, and `development` or + `development/*` for a subtree, which mean the same thing - and + `from_pattern/1` turns one into the scope it grants. + + Coverage is closed downward: a scope that covers a workspace covers + everything under that workspace too. That is what makes it sound to + authorise a whole scope - setting a secret for one, say - by asking + only whether a grant covers the scope's own name. + """ + + @typedoc """ + A scope, or `:never` - the scope of a pattern that grants nothing, + which covers no workspace at all. + """ + @type t :: String.t() | :never + + @doc """ + Whether `scope` covers `name`: the workspace is the scope itself, or + lies under it. + """ + @spec covers?(t(), String.t()) :: boolean() + def covers?(:never, _name), do: false + def covers?("", _name), do: true + + def covers?(scope, name) when is_binary(scope) and is_binary(name), + do: name == scope or String.starts_with?(name, scope <> "/") + + @doc """ + The scope a grant pattern grants. + + `*` is the root scope, and a trailing `/*` is optional, so + `development/*` and `development` both grant the `development` + subtree - the workspace itself and everything under it. + + Anything that names nothing - an empty pattern, or a bare `/*` - grants + `:never` rather than the root, so a stored pattern that means nothing + can't come to mean everything. + """ + @spec from_pattern(term()) :: t() + def from_pattern("*"), do: "" + + def from_pattern(pattern) when is_binary(pattern) do + case String.replace_suffix(pattern, "/*", "") do + "" -> :never + scope -> scope + end + end + + def from_pattern(_pattern), do: :never + + @doc """ + Whether a pattern names a scope - false for one that grants nothing. + """ + @spec valid_pattern?(term()) :: boolean() + def valid_pattern?(pattern), do: from_pattern(pattern) != :never +end diff --git a/server/test/coflux/scopes_test.exs b/server/test/coflux/scopes_test.exs new file mode 100644 index 00000000..b65cdae1 --- /dev/null +++ b/server/test/coflux/scopes_test.exs @@ -0,0 +1,76 @@ +defmodule Coflux.ScopesTest do + use ExUnit.Case, async: true + + alias Coflux.Scopes + + describe "covers?/2" do + test "a scope covers itself and everything under it" do + assert Scopes.covers?("development", "development") + assert Scopes.covers?("development", "development/joe") + assert Scopes.covers?("development", "development/joe/feature-1") + assert Scopes.covers?("development/joe", "development/joe/feature-1") + end + + test "the root scope covers every workspace" do + assert Scopes.covers?("", "development") + assert Scopes.covers?("", "development/joe") + end + + test "names are compared a segment at a time" do + refute Scopes.covers?("development", "development-2") + refute Scopes.covers?("development", "development2/joe") + refute Scopes.covers?("development/joe", "development/joel") + end + + test "a scope doesn't cover what's above it, or a sibling" do + refute Scopes.covers?("development/joe", "development") + refute Scopes.covers?("development", "") + refute Scopes.covers?("development/joe", "development/sam") + refute Scopes.covers?("development", "production") + end + + test "the scope of a pattern granting nothing covers nothing" do + refute Scopes.covers?(:never, "development") + refute Scopes.covers?(:never, "") + end + end + + describe "from_pattern/1" do + test "a bare name and a wildcard grant the same subtree" do + assert Scopes.from_pattern("development") == "development" + assert Scopes.from_pattern("development/*") == "development" + assert Scopes.from_pattern("development/joe/*") == "development/joe" + end + + test "a wildcard grant includes the workspace it names" do + scope = Scopes.from_pattern("development/*") + + assert Scopes.covers?(scope, "development") + assert Scopes.covers?(scope, "development/joe") + end + + test "'*' grants the root scope" do + assert Scopes.from_pattern("*") == "" + assert Scopes.covers?(Scopes.from_pattern("*"), "anything/at/all") + end + + test "a pattern that names nothing grants nothing, not everything" do + assert Scopes.from_pattern("") == :never + assert Scopes.from_pattern("/*") == :never + assert Scopes.from_pattern(nil) == :never + assert Scopes.from_pattern(["development"]) == :never + end + end + + describe "valid_pattern?/1" do + test "accepts what names a scope, rejects what doesn't" do + assert Scopes.valid_pattern?("*") + assert Scopes.valid_pattern?("development") + assert Scopes.valid_pattern?("development/*") + + refute Scopes.valid_pattern?("") + refute Scopes.valid_pattern?("/*") + refute Scopes.valid_pattern?(nil) + end + end +end diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 1f44415e..f33c9f6b 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -87,6 +87,21 @@ def test_token_outlives_rotation_and_restart(self, token_server): token_server.restart(timeout=30) assert _discover(port, project_id, created["token"]) == (200, ["*"]) + def _set_secret(self, port, project_id, token, scope): + """The status of setting a secret for a scope - a convenient probe + for what a grant covers.""" + try: + api_post( + port, + project_id, + "set_secret", + token=token, + body={"name": "key", "scope": scope, "value": "v"}, + ) + return 200 + except urllib.error.HTTPError as e: + return e.code + def test_setting_a_secret_takes_operator_access_to_its_scope(self, token_server): """A token for 'development/*' can set a secret for a development workspace, but not for the project or for production.""" @@ -95,21 +110,65 @@ def test_setting_a_secret_takes_operator_access_to_its_scope(self, token_server) restricted = _create(port, project_id, name="dev", workspaces=["development/*"]) def set_secret(scope): + return self._set_secret(port, project_id, restricted["token"], scope) + + assert set_secret("development/joe") == 200 + assert set_secret("") == 403 + assert set_secret("production") == 403 + + def test_a_grant_covers_the_workspace_it_names_and_everything_under_it( + self, token_server + ): + """A pattern grants a subtree: 'development' and 'development/*' + both cover 'development' itself and everything below it. A sibling + that merely starts with the same characters isn't below it.""" + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + bare = _create(port, project_id, name="bare", workspaces=["development"]) + wildcard = _create(port, project_id, name="wild", workspaces=["development/*"]) + + for created in [bare, wildcard]: + token = created["token"] + + assert self._set_secret(port, project_id, token, "development") == 200 + assert self._set_secret(port, project_id, token, "development/joe") == 200 + assert ( + self._set_secret(port, project_id, token, "development/joe/feature-1") + == 200 + ) + + assert self._set_secret(port, project_id, token, "development-2") == 403 + assert self._set_secret(port, project_id, token, "") == 403 + + def test_a_token_cannot_be_granted_more_than_its_creator_has(self, token_server): + """A token can hand on a scope within its own, and no more.""" + port = token_server.port + project_id = f"tok-{uuid.uuid4().hex[:8]}" + parent = _create(port, project_id, name="parent", workspaces=["development"]) + + def create(workspaces): try: - api_post( + _create( port, project_id, - "set_secret", - token=restricted["token"], - body={"name": "key", "scope": scope, "value": "v"}, + token=parent["token"], + name="child", + workspaces=workspaces, ) return 200 except urllib.error.HTTPError as e: return e.code - assert set_secret("development/joe") == 200 - assert set_secret("") == 403 - assert set_secret("production") == 403 + assert create(["development"]) == 200 + assert create(["development/*"]) == 200 + assert create(["development/joe"]) == 200 + + assert create(["*"]) == 403 + assert create(["production"]) == 403 + assert create(["development-2"]) == 403 + + # A pattern that names no scope is rejected outright. + assert create([""]) == 400 def test_only_the_creator_or_full_access_can_revoke(self, token_server): """Who created a token is kept with it, and resolved to a principal From 4351f7e71ba95690e27ba8a77109fc59d1199923 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 08:22:48 +0100 Subject: [PATCH 08/21] Fix flaky tests --- server/test/coflux/admin_secrets_test.exs | 34 ++++++++++++++--------- server/test/coflux/admin_tokens_test.exs | 8 ------ server/test/test_helper.exs | 8 ++++++ 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/server/test/coflux/admin_secrets_test.exs b/server/test/coflux/admin_secrets_test.exs index 6b0ca716..3bf28329 100644 --- a/server/test/coflux/admin_secrets_test.exs +++ b/server/test/coflux/admin_secrets_test.exs @@ -9,13 +9,6 @@ defmodule Coflux.AdminSecretsTest do @project "proj" @by %{type: "user", external_id: "user-1"} - setup_all do - previous = :persistent_term.get(:coflux_secret, nil) - :persistent_term.put(:coflux_secret, "test-secret") - on_exit(fn -> :persistent_term.put(:coflux_secret, previous) end) - :ok - end - setup do {:ok, db} = Sqlite3.open(":memory:") :ok = Migrations.run(db, "admin") @@ -121,14 +114,29 @@ defmodule Coflux.AdminSecretsTest do assert {:error, {:secret_invalid, "not-json"}} = Secrets.resolve_launcher(db, @project, "dev", %{credentials_secret: "not-json"}) end +end - test "without a server secret nothing can be stored or read", %{db: db} do +defmodule Coflux.AdminSecretsWithoutServerSecretTest do + # Takes the server secret away, which every other test's fixtures read, + # so it can't run alongside them. + use ExUnit.Case, async: false + + alias Coflux.Admin.Secrets + alias Coflux.Store.Migrations + alias Exqlite.Sqlite3 + + setup do + previous = :persistent_term.get(:coflux_secret, nil) :persistent_term.put(:coflux_secret, nil) + on_exit(fn -> :persistent_term.put(:coflux_secret, previous) end) + + {:ok, db} = Sqlite3.open(":memory:") + :ok = Migrations.run(db, "admin") + {:ok, db: db} + end - try do - assert {:error, :no_secret} = Secrets.set(db, @project, "", "x", "v", nil) - after - :persistent_term.put(:coflux_secret, "test-secret") - end + test "without a server secret nothing can be stored or read", %{db: db} do + assert {:error, :no_secret} = Secrets.set(db, "proj", "", "x", "v", nil) + assert {:error, :no_secret} = Secrets.resolve(db, "proj", "dev", "x") end end diff --git a/server/test/coflux/admin_tokens_test.exs b/server/test/coflux/admin_tokens_test.exs index 136eb49a..f9387410 100644 --- a/server/test/coflux/admin_tokens_test.exs +++ b/server/test/coflux/admin_tokens_test.exs @@ -7,14 +7,6 @@ defmodule Coflux.AdminTokensTest do alias Coflux.Store.Migrations alias Exqlite.Sqlite3 - # Creating a token signs it with the server secret. - setup_all do - previous = :persistent_term.get(:coflux_secret, nil) - :persistent_term.put(:coflux_secret, "test-secret") - on_exit(fn -> :persistent_term.put(:coflux_secret, previous) end) - :ok - end - # An orchestration database as it was before tokens moved: at version 5, # with a token, its principal, and a user principal that created it. defp legacy_orchestration_db do diff --git a/server/test/test_helper.exs b/server/test/test_helper.exs index 869559e7..af9e86f4 100644 --- a/server/test/test_helper.exs +++ b/server/test/test_helper.exs @@ -1 +1,9 @@ ExUnit.start() + +# The server secret is global state (`:persistent_term`, set from the +# environment at startup), and signing a service token or encrypting a +# secret reads it. The application isn't started under test, so set one +# for the whole run - a test that needs it to be a particular value, or +# absent, has to own the global while nothing else is running, which +# means `async: false`. +:persistent_term.put(:coflux_secret, "test-secret") From c5bf73b2718b2a36a870fb0e874f2cd4dc2e54e0 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 09:37:04 +0100 Subject: [PATCH 09/21] Tidy/fix definition of scopes --- cli/CHANGELOG.md | 2 +- cli/cmd/coflux/secrets.go | 84 +++++----- cli/cmd/coflux/tokens.go | 13 +- cli/internal/api/client.go | 34 +++-- docs/docs/authentication.md | 4 +- docs/docs/cli_reference.md | 4 +- docs/docs/pools.md | 56 +++++-- server/CHANGELOG.md | 4 +- server/lib/coflux/admin/secrets.ex | 23 ++- server/lib/coflux/handlers/api.ex | 64 +++----- server/lib/coflux/orchestration.ex | 8 +- server/lib/coflux/orchestration/server.ex | 102 +++++++++---- .../orchestration/server/permissions.ex | 11 +- server/lib/coflux/scopes.ex | 143 ++++++++++++------ server/test/coflux/admin_secrets_test.exs | 35 +++-- server/test/coflux/scopes_test.exs | 131 ++++++++++------ tests/support/cli.py | 16 +- tests/test_pools.py | 12 +- tests/test_secrets.py | 54 +++++-- tests/test_tokens.py | 63 ++++---- 20 files changed, 523 insertions(+), 340 deletions(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 004457bd..3b619733 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -4,7 +4,7 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`. - Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it. -- Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. +- Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. Each secret is set for one or more workspace patterns, given as a required `--workspaces`, in the same language `tokens create --workspaces` uses. ## 0.12.0 diff --git a/cli/cmd/coflux/secrets.go b/cli/cmd/coflux/secrets.go index 8e6b0a30..f870233d 100644 --- a/cli/cmd/coflux/secrets.go +++ b/cli/cmd/coflux/secrets.go @@ -6,6 +6,7 @@ import ( "io" "os" "sort" + "strings" "github.com/spf13/cobra" ) @@ -17,48 +18,53 @@ var secretsCmd = &cobra.Command{ or environment variables for workers - and that never appear in a pool's configuration. -A secret is set for a scope: a workspace name, or a prefix of one. A secret for -'development' applies to 'development/joe', and the nearest scope wins. Without ---scope or --global, the scope is the current workspace.`, +Every secret is set for one or more workspaces, given as --workspaces: a +workspace name for that one alone, 'development/*' for those under it, or '*' +for all of them. The '*' spans any depth, so 'development/*' reaches +'development/joe/feature-1' too, but it doesn't include 'development' itself - +give both ('development,development/*') for that. + +Where scopes overlap the nearest wins: an exact workspace beats a longer +prefix, which beats a shorter one, which beats '*'.`, } var ( - secretsScope string - secretsGlobal bool - secretsFromEnv string - secretsFromFile string + secretsWorkspaces string + secretsFromEnv string + secretsFromFile string ) func init() { for _, cmd := range []*cobra.Command{secretsSetCmd, secretsDeleteCmd} { - cmd.Flags().StringVar(&secretsScope, "scope", "", "Workspace name, or prefix, the secret applies to (default: the current workspace)") - cmd.Flags().BoolVar(&secretsGlobal, "global", false, "Apply to every workspace in the project") + cmd.Flags().StringVar(&secretsWorkspaces, "workspaces", "", "Comma-separated workspace patterns the secret applies to") + cmd.MarkFlagRequired("workspaces") } secretsSetCmd.Flags().StringVar(&secretsFromEnv, "from-env", "", "Read the value from this environment variable") secretsSetCmd.Flags().StringVar(&secretsFromFile, "from-file", "", "Read the value from this file") secretsCmd.AddCommand(secretsSetCmd, secretsListCmd, secretsDeleteCmd) } -// secretScope is the scope a secret is set or deleted in: "" for the whole -// project, else a workspace name or prefix. -func secretScope() (string, error) { - if secretsGlobal && secretsScope != "" { - return "", fmt.Errorf("--scope and --global can't both be given") - } - if secretsGlobal { - return "", nil +// secretWorkspaces is the patterns a secret is set for or deleted from. The +// secret is stored once per pattern, so each is listed and rotated on its own. +func secretWorkspaces() ([]string, error) { + var workspaces []string + for _, part := range strings.Split(secretsWorkspaces, ",") { + if part = strings.TrimSpace(part); part != "" { + workspaces = append(workspaces, part) + } } - if secretsScope != "" { - return secretsScope, nil + if len(workspaces) == 0 { + return nil, fmt.Errorf("--workspaces can't be empty") } - return requireWorkspace() + return workspaces, nil } -func describeScope(scope string) string { - if scope == "" { - return "all workspaces" +func describeWorkspaces(workspaces []string) string { + quoted := make([]string, len(workspaces)) + for i, workspace := range workspaces { + quoted[i] = fmt.Sprintf("'%s'", workspace) } - return fmt.Sprintf("'%s'", scope) + return strings.Join(quoted, ", ") } // secrets set @@ -71,8 +77,9 @@ var secretsSetCmd = &cobra.Command{ The value is read from stdin, so it never appears on the command line or in shell history: - printf '%s' "$API_KEY" | coflux secrets set api-key - aws configure export-credentials --profile sandbox | coflux secrets set aws-sandbox + printf '%s' "$API_KEY" | coflux secrets set api-key --workspaces '*' + aws configure export-credentials --profile sandbox | + coflux secrets set aws-sandbox --workspaces 'production/*' Or from an environment variable (--from-env) or a file (--from-file). A single trailing newline is dropped, so 'echo' works too.`, @@ -83,7 +90,7 @@ trailing newline is dropped, so 'echo' works too.`, func runSecretsSet(cmd *cobra.Command, args []string) error { name := args[0] - scope, err := secretScope() + workspaces, err := secretWorkspaces() if err != nil { return err } @@ -98,12 +105,14 @@ func runSecretsSet(cmd *cobra.Command, args []string) error { return err } - result, err := client.SetSecret(cmd.Context(), scope, name, value) + result, err := client.SetSecret(cmd.Context(), workspaces, name, value) if err != nil { return err } - fmt.Printf("Set secret '%s' (version %d) for %s.\n", result.Name, result.Version, describeScope(result.Scope)) + for _, secret := range result.Secrets { + fmt.Printf("Set secret '%s' (version %d) for '%s'.\n", result.Name, secret.Version, secret.Workspaces) + } return nil } @@ -150,7 +159,7 @@ func readSecretValue() (string, error) { var secretsListCmd = &cobra.Command{ Use: "list", Short: "List secrets", - Long: "List the project's secrets: their names, scopes and versions. Never their values.", + Long: "List the project's secrets: their names, workspaces and versions. Never their values.", RunE: runSecretsList, } @@ -184,24 +193,20 @@ func runSecretsList(cmd *cobra.Command, args []string) error { var rows [][]string for _, s := range secrets { - scope := getString(s, "scope") - if scope == "" { - scope = "(all)" - } by := "-" if principal, ok := s["updatedBy"].(map[string]any); ok { by = fmt.Sprintf("%s %s", getString(principal, "type"), getString(principal, "externalId")) } rows = append(rows, []string{ getString(s, "name"), - scope, + getString(s, "scope"), fmt.Sprintf("%d", int(getFloat64(s, "version"))), formatTimestamp(getInt64(s, "updatedAt")), by, }) } - printTable([]string{"Name", "Scope", "Version", "Updated", "By"}, rows) + printTable([]string{"Name", "Workspaces", "Version", "Updated", "By"}, rows) return nil } @@ -217,7 +222,7 @@ var secretsDeleteCmd = &cobra.Command{ func runSecretsDelete(cmd *cobra.Command, args []string) error { name := args[0] - scope, err := secretScope() + workspaces, err := secretWorkspaces() if err != nil { return err } @@ -227,10 +232,11 @@ func runSecretsDelete(cmd *cobra.Command, args []string) error { return err } - if err := client.DeleteSecret(cmd.Context(), scope, name); err != nil { + result, err := client.DeleteSecret(cmd.Context(), workspaces, name) + if err != nil { return err } - fmt.Printf("Deleted secret '%s' for %s.\n", name, describeScope(scope)) + fmt.Printf("Deleted secret '%s' for %s.\n", name, describeWorkspaces(result.Workspaces)) return nil } diff --git a/cli/cmd/coflux/tokens.go b/cli/cmd/coflux/tokens.go index e0add58d..fb951c26 100644 --- a/cli/cmd/coflux/tokens.go +++ b/cli/cmd/coflux/tokens.go @@ -97,11 +97,14 @@ var tokensCreateCmd = &cobra.Command{ The token value is displayed only once. Make sure to copy it. -Use --workspaces to restrict the token to specific workspaces. A pattern grants -a workspace and everything under it, so 'development' and 'development/*' both -cover 'development' and 'development/joe'. Use '*' for every workspace. A token -can't be given broader access than the one creating it. If omitted, the token -inherits the access of the token creating it.`, +Use --workspaces to restrict the token to specific workspaces. A pattern selects +one workspace ('development'), the workspaces under it ('development/*', at any +depth, but not 'development' itself), or all of them ('*'). Give several +comma-separated for the union: 'development,development/*'. + +A token can't be given broader access than the one creating it - each pattern +has to sit wholly inside one the creator holds. If omitted, the token inherits +the access of the token creating it.`, RunE: runTokensCreate, } diff --git a/cli/internal/api/client.go b/cli/internal/api/client.go index 348a1090..83da6787 100644 --- a/cli/internal/api/client.go +++ b/cli/internal/api/client.go @@ -369,16 +369,19 @@ func (c *Client) RevokeToken(ctx context.Context, externalID string) error { // Secrets API -// SetSecretResult is what the server says about a secret it has set. +// SetSecretResult is what the server says about the secrets it has set - +// one entry per workspace pattern the value was stored for. type SetSecretResult struct { Name string `json:"name"` - Scope string `json:"scope"` - Version int `json:"version"` + Secrets []struct { + Workspaces string `json:"workspaces"` + Version int `json:"version"` + } `json:"secrets"` } -// SetSecret sets a secret's value for a scope ("" for the whole project). -func (c *Client) SetSecret(ctx context.Context, scope, name, value string) (*SetSecretResult, error) { - body := map[string]any{"name": name, "value": value, "scope": scope} +// SetSecret sets a secret's value for each of the given workspace patterns. +func (c *Client) SetSecret(ctx context.Context, workspaces []string, name, value string) (*SetSecretResult, error) { + body := map[string]any{"name": name, "value": value, "workspaces": workspaces} var result SetSecretResult if _, err := c.post(ctx, "/api/set_secret", body, &result); err != nil { return nil, err @@ -386,11 +389,20 @@ func (c *Client) SetSecret(ctx context.Context, scope, name, value string) (*Set return &result, nil } -// DeleteSecret deletes a secret from a scope ("" for the whole project). -func (c *Client) DeleteSecret(ctx context.Context, scope, name string) error { - body := map[string]any{"name": name, "scope": scope} - _, err := c.post(ctx, "/api/delete_secret", body, nil) - return err +// DeleteSecretResult is the workspace patterns a secret was deleted from - +// those of the requested ones it was actually set for. +type DeleteSecretResult struct { + Workspaces []string `json:"workspaces"` +} + +// DeleteSecret deletes a secret from each of the given workspace patterns. +func (c *Client) DeleteSecret(ctx context.Context, workspaces []string, name string) (*DeleteSecretResult, error) { + body := map[string]any{"name": name, "workspaces": workspaces} + var result DeleteSecretResult + if _, err := c.post(ctx, "/api/delete_secret", body, &result); err != nil { + return nil, err + } + return &result, nil } // ListSecrets lists the project's secrets: names, scopes and versions. diff --git a/docs/docs/authentication.md b/docs/docs/authentication.md index 92173b0d..d3552c91 100644 --- a/docs/docs/authentication.md +++ b/docs/docs/authentication.md @@ -42,7 +42,9 @@ Service tokens can be created and scoped to specific workspaces, making them sui coflux tokens create --name "CI" --workspaces "production/*" ``` -A pattern grants a workspace and everything under it, following the `/` in workspace names. So `production` and `production/*` both cover `production` itself and `production/eu`, but neither covers `production-2`. Use `*` for every workspace. A token can't be created with broader access than the token creating it has, and one created without `--workspaces` inherits the creating token's access. +A pattern selects workspaces: `production` is that workspace alone, `production/*` is the workspaces under it, and `*` is all of them. Give several, comma-separated, for the union — `production,production/*` covers a workspace and everything under it. The `*` spans any depth, so `production/*` reaches `production/eu/batch`, and it follows the `/` in names, so neither pattern ever selects `production-2`. + +A token can't be created with broader access than the token creating it has: each pattern it asks for has to sit wholly inside one the creator holds. A token created without `--workspaces` inherits the creating token's access. Service tokens require `COFLUX_SECRET` to be configured on the server. This secret is used to derive per-project signing keys. diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index 68361144..83160113 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -218,9 +218,9 @@ See [pools](./pools.md) for launcher-specific fields. | Command | Description | |---------|-------------| -| `secrets set ` | Set a secret's value from stdin (`--from-env`, `--from-file`, `--scope`, `--global`) | +| `secrets set ` | Set a secret's value from stdin (`--workspaces`, `--from-env`, `--from-file`) | | `secrets list` | List secrets (names, scopes and versions) | -| `secrets delete ` | Delete a secret (`--scope`, `--global`) | +| `secrets delete ` | Delete a secret (`--workspaces`) | ## `coflux inputs` diff --git a/docs/docs/pools.md b/docs/docs/pools.md index e28b45a7..eea454f1 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -71,7 +71,7 @@ coflux pools create mypool --type kubernetes \ When the Coflux server runs inside Kubernetes, it automatically uses in-cluster authentication. For external servers, provide the API server URL and name a [secret](#secrets) holding a bearer token: ```bash -coflux secrets set k8s-token < /path/to/token +coflux secrets set k8s-token --workspaces '*' < /path/to/token coflux pools create mypool --type kubernetes \ --set image=myorg/myapp:latest \ @@ -137,7 +137,7 @@ whose value is JSON in the shape the AWS CLI produces, so a profile's credentials can be stored directly: ```bash -aws configure export-credentials --profile sandbox | coflux secrets set aws-sandbox +aws configure export-credentials --profile sandbox |\n coflux secrets set aws-sandbox --workspaces 'production/*' coflux pools update mypool --set credentialsSecret=aws-sandbox ``` @@ -195,30 +195,56 @@ encrypted, and referred to by name. A pool's configuration, `pools get`, and The value is read from stdin, so it never appears on the command line: ```bash -printf '%s' "$OPENAI_API_KEY" | coflux secrets set openai +printf '%s' "$OPENAI_API_KEY" | coflux secrets set openai --workspaces 'development/*' coflux pools update mypool --set envSecrets.OPENAI_API_KEY=openai ``` `--from-env` and `--from-file` read it from an environment variable or a file -instead. `secrets list` shows names, scopes and versions, never values, and +instead. `secrets list` shows names, workspaces and versions, never values, and `secrets delete` removes one. -A secret applies to a _scope_: a workspace name, or a prefix of one. A secret -set for `development` applies to `development/joe`, and the nearest scope wins, -so `development/joe` can override it. By default the scope is the current -workspace; `--scope` names another, and `--global` applies it to every -workspace. Scopes follow workspace names, not what a workspace inherits from: -a workspace that inherits from `production` doesn't see production's secrets. +### Which workspaces a secret applies to -Setting a secret takes operator access to its scope, and `--global` takes -access to every workspace. Setting one again replaces its value and bumps its -version; workers already running keep the value they were launched with. -Values are encrypted with a key derived from `COFLUX_SECRET`, which must be -configured for secrets to be used. +`--workspaces` is required, and takes the same patterns a token's access does: + +| Pattern | Selects | +| --- | --- | +| `development` | that workspace, and nothing else | +| `development/*` | the workspaces under `development/` — but not `development` itself | +| `*` | every workspace in the project | + +The `*` is not a glob: `development/*` reaches `development/joe` and +`development/joe/feature-1` alike, because it matches a prefix of the name +rather than one level of it. For a workspace *and* everything under it, give +both: `--workspaces 'development,development/*'`. + +Several patterns, comma-separated, store the value once for each — so they are +listed, rotated and deleted separately. Patterns follow workspace names, not +what a workspace inherits from: a workspace that inherits results from +`production` doesn't see production's secrets, and `development` never selects +`development-2`. + +Where patterns overlap the nearest wins: an exact workspace beats a longer +prefix, which beats a shorter one, which beats `*`. + +Setting a secret takes access containing every pattern given, whole — a token +for `development/joe` can't set a secret for `development/*`, because that +would reach workspaces it has no access to. Setting one again replaces its +value and bumps its version; workers already running keep the value they were +launched with. Values are encrypted with a key derived from `COFLUX_SECRET`, +which must be configured for secrets to be used. A pool that names a secret its workspace can't see is refused when it is created, updated, or imported. +### Who can read a secret + +Reading is bounded by the patterns, not by the access it took to set the +value: a pool launches its workers with the secrets it names, so anyone who +can edit a pool in a workspace can obtain the value of every secret that +applies there. Give a secret no more workspaces than the people who should be +able to read it — a secret set for `*` is readable by everyone in the project. + ## Managing pools ```bash diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 3c9e470e..6ae0a6be 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -2,7 +2,9 @@ Changes: -- A workspace pattern on a token now grants the workspace it names *and everything under it*, so `development/*` covers `development` itself (as documented, but not previously implemented), and a plain `development` covers `development/joe`. Previously a plain pattern matched one workspace exactly, and `development/*` matched everything below `development` but not `development` itself. Tokens already issued widen accordingly, always to workspaces under a name the token already held. Patterns that name nothing (an empty pattern) are now rejected when creating a token. +- Workspace patterns mean one thing everywhere now. A pattern selects a workspace (`development`), the workspaces under it (`development/*`, at any depth, but not `development` itself), or all of them (`*`) — the rule tokens already used, now used for a secret's workspaces too, where a bare name previously selected everything beneath it as well. Patterns that name nothing are rejected. +- Setting a secret takes access containing every pattern given, whole, rather than access to any one workspace the pattern reaches — so a token for `staging` can no longer set a secret reaching `staging/feature-1`. +- Secrets are set for one or more workspace patterns (`workspaces` on `set_secret` and `delete_secret`, replacing `scope`), stored once per pattern. Where patterns overlap, the nearest wins: an exact workspace, then a longer prefix, then a shorter one, then `*`. ## 0.12.0 diff --git a/server/lib/coflux/admin/secrets.ex b/server/lib/coflux/admin/secrets.ex index 6c51c7d8..d133b87d 100644 --- a/server/lib/coflux/admin/secrets.ex +++ b/server/lib/coflux/admin/secrets.ex @@ -9,11 +9,12 @@ defmodule Coflux.Admin.Secrets do value and bumps the version; deleting it removes it. Nothing older is kept, so a rotated or deleted value is gone from here. - A secret is set for a scope, as `Coflux.Scopes` defines one: a secret - for `development` applies to `development/joe`, and the nearest scope - wins. The root scope is the whole project. This follows the naming - hierarchy rather than the base-workspace chain: a workspace that - inherits results from `production` doesn't inherit its secrets. + A secret is set for a scope, as `Coflux.Scopes` defines one: `*` for + every workspace, `development/*` for those under `development/`, or a + workspace name for that one alone. Where scopes overlap the nearest one + wins. This follows the naming hierarchy rather than the base-workspace + chain: a workspace that inherits results from `production` doesn't + inherit its secrets. Pools name secrets in fields that take nothing else - `tokenSecret`, `credentialsSecret`, `envSecrets` - and the values are resolved for a @@ -28,16 +29,10 @@ defmodule Coflux.Admin.Secrets do @max_value_bytes 65_536 @name_regex ~r/^[a-z0-9][a-z0-9_-]{0,63}$/i - @scope_regex ~r/^[a-z0-9][a-z0-9_\/-]{0,99}$/i def valid_name?(name), do: is_binary(name) and Regex.match?(@name_regex, name) - def valid_scope?(""), do: true - - def valid_scope?(scope), - do: - is_binary(scope) and Regex.match?(@scope_regex, scope) and - not String.ends_with?(scope, "/") + defdelegate valid_scope?(scope), to: Coflux.Scopes, as: :valid? def valid_value?(value), do: is_binary(value) and byte_size(value) <= @max_value_bytes @@ -302,7 +297,7 @@ defmodule Coflux.Admin.Secrets do # --- Storage and crypto --- - # Nearest scope wins: the longest that covers the workspace. + # Nearest scope wins: of those covering the workspace, the most specific. defp find(db, workspace_name, name) do {:ok, rows} = Store.query( @@ -313,7 +308,7 @@ defmodule Coflux.Admin.Secrets do rows |> Enum.filter(fn {scope, _, _, _} -> scope_applies?(scope, workspace_name) end) - |> Enum.max_by(fn {scope, _, _, _} -> byte_size(scope) end, fn -> nil end) + |> Enum.max_by(fn {scope, _, _, _} -> Coflux.Scopes.specificity(scope) end, fn -> nil end) |> case do nil -> {:error, :not_found} row -> {:ok, row} diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 7423a09f..79e8d998 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -73,22 +73,14 @@ defmodule Coflux.Handlers.Api do end end - # Whether the caller can grant the requested access: every scope asked - # for has to sit within one the caller already has. Scopes are closed - # downward, so a caller's scope covering the root of a requested one - # covers all of it. + # Whether the caller can grant the requested access: each scope asked for + # has to be contained whole by one the caller holds. Holding a workspace + # inside a scope is not holding the scope. defp workspaces_covered?(:all, _requested), do: true defp workspaces_covered?(_caller, nil), do: true - defp workspaces_covered?(caller_patterns, requested) do - caller_scopes = Enum.map(caller_patterns, &Scopes.from_pattern/1) - - Enum.all?(requested, fn pattern -> - case Scopes.from_pattern(pattern) do - :never -> false - scope -> Enum.any?(caller_scopes, &Scopes.covers?(&1, scope)) - end - end) + defp workspaces_covered?(caller_scopes, requested) do + Enum.all?(requested, &Scopes.contains_any?(caller_scopes, &1)) end defp handle(req, "GET", ["discover"], _project_id, %{workspaces: workspaces}) do @@ -985,27 +977,26 @@ defmodule Coflux.Handlers.Api do end defp handle(req, "POST", ["set_secret"], project_id, access) do - case read_arguments( - req, - %{ - name: {"name", &parse_secret_name/1}, - value: {"value", &parse_secret_value/1} - }, - %{scope: {"scope", &parse_secret_scope/1}} - ) do + case read_arguments(req, %{ + name: {"name", &parse_secret_name/1}, + value: {"value", &parse_secret_value/1}, + workspaces: {"workspaces", &parse_workspaces/1} + }) do {:ok, arguments, req} -> case Orchestration.set_secret( project_id, - Map.get(arguments, :scope, ""), + arguments.workspaces, arguments.name, arguments.value, access ) do - {:ok, secret} -> + {:ok, secrets} -> json_response(req, %{ - "name" => secret.name, - "scope" => secret.scope, - "version" => secret.version + "name" => arguments.name, + "secrets" => + Enum.map(secrets, fn secret -> + %{"workspaces" => secret.scope, "version" => secret.version} + end) }) {:error, :forbidden} -> @@ -1023,19 +1014,18 @@ defmodule Coflux.Handlers.Api do end defp handle(req, "POST", ["delete_secret"], project_id, access) do - case read_arguments( - req, - %{name: {"name", &parse_secret_name/1}}, - %{scope: {"scope", &parse_secret_scope/1}} - ) do + case read_arguments(req, %{ + name: {"name", &parse_secret_name/1}, + workspaces: {"workspaces", &parse_workspaces/1} + }) do {:ok, arguments, req} -> case Orchestration.delete_secret( project_id, - Map.get(arguments, :scope, ""), + arguments.workspaces, arguments.name, access ) do - :ok -> :cowboy_req.reply(204, req) + {:ok, deleted} -> json_response(req, %{"workspaces" => deleted}) {:error, :not_found} -> json_error_response(req, "not_found", status: 404) {:error, :forbidden} -> json_error_response(req, "forbidden", status: 403) end @@ -1103,8 +1093,8 @@ defmodule Coflux.Handlers.Api do # Helper functions for handle/5 clauses defp parse_workspaces(value) when is_list(value) do - if Enum.all?(value, &Scopes.valid_pattern?/1) do - {:ok, value} + if value != [] and Enum.all?(value, &Scopes.valid?/1) do + {:ok, Enum.uniq(value)} else {:error, :invalid} end @@ -1700,10 +1690,6 @@ defmodule Coflux.Handlers.Api do if Coflux.Admin.Secrets.valid_name?(value), do: {:ok, value}, else: {:error, :invalid} end - defp parse_secret_scope(value) do - if Coflux.Admin.Secrets.valid_scope?(value), do: {:ok, value}, else: {:error, :invalid} - end - defp parse_secret_value(value) do if Coflux.Admin.Secrets.valid_value?(value), do: {:ok, value}, else: {:error, :invalid} end diff --git a/server/lib/coflux/orchestration.ex b/server/lib/coflux/orchestration.ex index 81158540..da11eb98 100644 --- a/server/lib/coflux/orchestration.ex +++ b/server/lib/coflux/orchestration.ex @@ -59,12 +59,12 @@ defmodule Coflux.Orchestration do call_server(project_id, {:archive_workspace, workspace_id, access}) end - def set_secret(project_id, scope, name, value, access \\ nil) do - call_server(project_id, {:set_secret, scope, name, value, access}) + def set_secret(project_id, scopes, name, value, access \\ nil) do + call_server(project_id, {:set_secret, scopes, name, value, access}) end - def delete_secret(project_id, scope, name, access \\ nil) do - call_server(project_id, {:delete_secret, scope, name, access}) + def delete_secret(project_id, scopes, name, access \\ nil) do + call_server(project_id, {:delete_secret, scopes, name, access}) end def get_pools(project_id, workspace_id) do diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index 74dbed9e..ae5c45ef 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -27,6 +27,7 @@ defmodule Coflux.Orchestration.Server do WorkspaceUpdated } + alias Coflux.Scopes alias Coflux.Store.{Epochs, Index} alias Coflux.Orchestration.{ @@ -318,37 +319,65 @@ defmodule Coflux.Orchestration.Server do # Secrets - defp dispatch_call({:set_secret, scope, name, value, access}, state) do - with :ok <- check_secret_scope_access(access, scope), - {:ok, secret} <- - Coflux.Admin.Secrets.set( - state.admin_db, - state.project_id, - scope, - name, - value, - principal_identity(state, access) - ) do - state = - Effects.emit(state, %SecretSet{ - scope: secret.scope, - name: secret.name, - version: secret.version, - created_at: secret.created_at, - updated_at: secret.updated_at, - updated_by: secret.updated_by - }) + # A secret is set for one or more scopes, each stored on its own. Access + # to every scope is checked before any is written, so a request that + # isn't wholly allowed changes nothing. + defp dispatch_call({:set_secret, scopes, name, value, access}, state) do + with :ok <- check_secret_scope_access(access, scopes) do + identity = principal_identity(state, access) + + Enum.reduce_while(scopes, {:reply, {:ok, []}, state}, fn scope, + {:reply, {:ok, secrets}, state} -> + case Coflux.Admin.Secrets.set( + state.admin_db, + state.project_id, + scope, + name, + value, + identity + ) do + {:ok, secret} -> + state = + Effects.emit(state, %SecretSet{ + scope: secret.scope, + name: secret.name, + version: secret.version, + created_at: secret.created_at, + updated_at: secret.updated_at, + updated_by: secret.updated_by + }) + + {:cont, {:reply, {:ok, secrets ++ [secret]}, state}} - {:reply, {:ok, secret}, state} + {:error, reason} -> + {:halt, {:reply, {:error, reason}, state}} + end + end) else {:error, reason} -> {:reply, {:error, reason}, state} end end - defp dispatch_call({:delete_secret, scope, name, access}, state) do - with :ok <- check_secret_scope_access(access, scope), - :ok <- Coflux.Admin.Secrets.delete(state.admin_db, scope, name) do - {:reply, :ok, Effects.emit(state, %SecretDeleted{scope: scope, name: name})} + # Deleting is done scope by scope: it is not an error for a secret to be + # absent from some of them, only from all of them. + defp dispatch_call({:delete_secret, scopes, name, access}, state) do + with :ok <- check_secret_scope_access(access, scopes) do + {state, deleted} = + Enum.reduce(scopes, {state, []}, fn scope, {state, deleted} -> + case Coflux.Admin.Secrets.delete(state.admin_db, scope, name) do + :ok -> + {Effects.emit(state, %SecretDeleted{scope: scope, name: name}), deleted ++ [scope]} + + {:error, :not_found} -> + {state, deleted} + end + end) + + if deleted == [] do + {:reply, {:error, :not_found}, state} + else + {:reply, {:ok, deleted}, state} + end else {:error, reason} -> {:reply, {:error, reason}, state} end @@ -2976,15 +3005,22 @@ defmodule Coflux.Orchestration.Server do # Private helper functions - # A secret in a scope is a secret for every workspace under it, so - # setting one takes operator access to the scope itself. Scopes are - # closed downward (see `Coflux.Scopes`), so a grant covering the scope's - # own name covers everything the secret can reach - asking about the - # name alone is enough. The root scope is covered only by a `*` grant. - defp check_secret_scope_access(nil, _scope), do: :ok + # A secret set for a scope is a secret for every workspace that scope + # selects, so setting one takes a grant that contains the scope whole. + # Holding one workspace inside it isn't enough - that is the difference + # between `Scopes.covers?/2` and `Scopes.contains?/2`. + defp check_secret_scope_access(nil, _scopes), do: :ok - defp check_secret_scope_access(access, scope) do - if Permissions.operator?(access[:workspaces], scope), do: :ok, else: {:error, :forbidden} + defp check_secret_scope_access(access, scopes) do + case access[:workspaces] do + :all -> + :ok + + granted -> + if Enum.all?(scopes, &Scopes.contains_any?(granted, &1)), + do: :ok, + else: {:error, :forbidden} + end end defp principal_identity(state, access) do diff --git a/server/lib/coflux/orchestration/server/permissions.ex b/server/lib/coflux/orchestration/server/permissions.ex index 8495f061..7ebe66e0 100644 --- a/server/lib/coflux/orchestration/server/permissions.ex +++ b/server/lib/coflux/orchestration/server/permissions.ex @@ -10,9 +10,10 @@ defmodule Coflux.Orchestration.Server.Permissions do Access arrives as the caller's grant - a token's workspace patterns, or a studio session's - and `nil` means an internal caller with no restriction. A pattern grants a scope of the workspace naming - hierarchy, which `Coflux.Scopes` defines, and an operator grant - additionally allows the management operations (creating workspaces, - editing pools, revoking tokens) that a plain workspace grant does not. + hierarchy, which `Coflux.Scopes` defines. There is one level of it: a + grant covering a workspace allows everything in that workspace, from + submitting a run to editing its pools. "Operator" here means only that + - holding the workspace - not a second, higher kind of grant. Also here: which workspaces a cache lookup may reach into, which is the same question of visibility asked of the workspace graph rather than of @@ -61,9 +62,7 @@ defmodule Coflux.Orchestration.Server.Permissions do def operator?(:all, _workspace), do: true - def operator?(patterns, workspace) do - Enum.any?(patterns, &Scopes.covers?(Scopes.from_pattern(&1), workspace)) - end + def operator?(scopes, workspace), do: Scopes.covers_any?(scopes, workspace) def check_operator_access(nil, _name), do: :ok diff --git a/server/lib/coflux/scopes.ex b/server/lib/coflux/scopes.ex index a4933b57..6a4e6c92 100644 --- a/server/lib/coflux/scopes.ex +++ b/server/lib/coflux/scopes.ex @@ -1,68 +1,117 @@ defmodule Coflux.Scopes do @moduledoc """ - The workspace naming hierarchy, and the one relation asked of it: - whether a scope covers a workspace. - - A scope is a workspace name, or a prefix of one along the `/` in the - name - `development` covers `development/joe` and `development/joe/x` - - and the root scope, `""`, covers every workspace. Names are compared a - segment at a time, so `development` doesn't cover `development-2`. - - Two things are defined in terms of this: which workspaces a secret set - for a scope applies to, and which workspaces a token's grant allows. A - grant is written as a pattern - `*` for the root, and `development` or - `development/*` for a subtree, which mean the same thing - and - `from_pattern/1` turns one into the scope it grants. - - Coverage is closed downward: a scope that covers a workspace covers - everything under that workspace too. That is what makes it sound to - authorise a whole scope - setting a secret for one, say - by asking - only whether a grant covers the scope's own name. - """ + Scopes: the patterns that select workspaces. + + A workspace name identifies one workspace - `development/joe`. A scope + selects a set of them: + + * `*` - every workspace + * `development/*` - every workspace under `development/`, but not + `development` itself + * `development` - that workspace, and nothing else + + The `*` is not a glob. `development/*` reaches `development/joe` and + `development/joe/feature-1` alike - it is a prefix of the name, not one + level of it. To select a workspace *and* everything under it, name both: + `development,development/*`. + + Several scopes given together - a token's grant, or the workspaces a + secret is set for - mean the union of what each selects. + + Two questions get asked of a scope, and they are not the same question: - @typedoc """ - A scope, or `:never` - the scope of a pattern that grants nothing, - which covers no workspace at all. + * `covers?/2` - does this scope select this workspace? Asked when a + caller acts on a workspace, and when a secret is resolved for one. + + * `contains?/2` - does this scope select everything that one does? + Asked when a caller hands authority on: setting a secret for a + scope, or minting a token for one. Selecting *part* of a scope is + not enough to give it away, and asking `covers?/2` there - treating + the scope as though it were a workspace name - is how a grant over + one workspace came to authorise a scope reaching others. """ - @type t :: String.t() | :never + + @scope_regex ~r/^[a-z0-9][a-z0-9_\/-]{0,99}$/i @doc """ - Whether `scope` covers `name`: the workspace is the scope itself, or - lies under it. + Whether `scope` selects the workspace called `name`. """ - @spec covers?(t(), String.t()) :: boolean() - def covers?(:never, _name), do: false - def covers?("", _name), do: true + @spec covers?(String.t(), String.t()) :: boolean() + def covers?("*", _name), do: true - def covers?(scope, name) when is_binary(scope) and is_binary(name), - do: name == scope or String.starts_with?(name, scope <> "/") + def covers?(scope, name) when is_binary(scope) and is_binary(name) do + case prefix_base(scope) do + nil -> name == scope + base -> String.starts_with?(name, base <> "/") + end + end @doc """ - The scope a grant pattern grants. + Whether `outer` selects everything `inner` selects. + + A scope contains itself, `*` contains everything, and `development/*` + contains `development/joe` and `development/joe/*` - but not + `development`, which it doesn't select. + """ + @spec contains?(String.t(), String.t()) :: boolean() + def contains?("*", _inner), do: true + def contains?(_outer, "*"), do: false - `*` is the root scope, and a trailing `/*` is optional, so - `development/*` and `development` both grant the `development` - subtree - the workspace itself and everything under it. + def contains?(outer, inner) when is_binary(outer) and is_binary(inner) do + case {prefix_base(outer), prefix_base(inner)} do + # An exact scope selects one workspace, so it can only contain the + # scope that selects the same one. + {nil, _} -> outer == inner + {base, nil} -> String.starts_with?(inner, base <> "/") + {base, inner_base} -> String.starts_with?(inner_base <> "/", base <> "/") + end + end + + @doc "Whether any of `scopes` selects the workspace called `name`." + @spec covers_any?([String.t()], String.t()) :: boolean() + def covers_any?(scopes, name), do: Enum.any?(scopes, &covers?(&1, name)) - Anything that names nothing - an empty pattern, or a bare `/*` - grants - `:never` rather than the root, so a stored pattern that means nothing - can't come to mean everything. + @doc "Whether any of `scopes` contains `inner` whole." + @spec contains_any?([String.t()], String.t()) :: boolean() + def contains_any?(scopes, inner), do: Enum.any?(scopes, &contains?(&1, inner)) + + @doc """ + Whether this is a scope: `*`, a workspace name, or one with `/*`. """ - @spec from_pattern(term()) :: t() - def from_pattern("*"), do: "" + @spec valid?(term()) :: boolean() + def valid?("*"), do: true - def from_pattern(pattern) when is_binary(pattern) do - case String.replace_suffix(pattern, "/*", "") do - "" -> :never - scope -> scope + def valid?(scope) when is_binary(scope) do + case prefix_base(scope) do + nil -> Regex.match?(@scope_regex, scope) and not String.ends_with?(scope, "/") + base -> base != "" and Regex.match?(@scope_regex, base) end end - def from_pattern(_pattern), do: :never + def valid?(_scope), do: false @doc """ - Whether a pattern names a scope - false for one that grants nothing. + How specific a scope is, for picking between those that all cover the + same workspace: an exact scope beats a prefix, a longer prefix beats a + shorter one, and `*` loses to everything. + + Only meaningful between scopes covering the same workspace - which is + why it is a number and not an ordering of scopes in general. """ - @spec valid_pattern?(term()) :: boolean() - def valid_pattern?(pattern), do: from_pattern(pattern) != :never + @spec specificity(String.t()) :: {non_neg_integer(), non_neg_integer()} + def specificity("*"), do: {0, 0} + + def specificity(scope) do + case prefix_base(scope) do + nil -> {2, byte_size(scope)} + base -> {1, byte_size(base)} + end + end + + # The name before a trailing `/*`, or nil when the scope has none. + defp prefix_base(scope) do + if String.ends_with?(scope, "/*") do + binary_part(scope, 0, byte_size(scope) - 2) + end + end end diff --git a/server/test/coflux/admin_secrets_test.exs b/server/test/coflux/admin_secrets_test.exs index 3bf28329..50cfc7f1 100644 --- a/server/test/coflux/admin_secrets_test.exs +++ b/server/test/coflux/admin_secrets_test.exs @@ -16,32 +16,37 @@ defmodule Coflux.AdminSecretsTest do end test "a value round-trips, and setting it again replaces it and bumps the version", %{db: db} do - assert {:ok, %{version: 1}} = Secrets.set(db, @project, "", "api-key", "first", @by) + assert {:ok, %{version: 1}} = Secrets.set(db, @project, "*", "api-key", "first", @by) assert {:ok, "first"} = Secrets.resolve(db, @project, "development", "api-key") assert {:ok, %{version: 2, updated_by: @by}} = - Secrets.set(db, @project, "", "api-key", "second", @by) + Secrets.set(db, @project, "*", "api-key", "second", @by) assert {:ok, "second"} = Secrets.resolve(db, @project, "development", "api-key") - assert {:ok, [%{name: "api-key", scope: "", version: 2}]} = Secrets.list(db) - assert :ok = Secrets.delete(db, "", "api-key") - assert {:error, :not_found} = Secrets.delete(db, "", "api-key") + assert {:ok, [%{name: "api-key", scope: "*", version: 2}]} = Secrets.list(db) + assert :ok = Secrets.delete(db, "*", "api-key") + assert {:error, :not_found} = Secrets.delete(db, "*", "api-key") assert {:error, :not_found} = Secrets.resolve(db, @project, "development", "api-key") end test "the nearest scope wins, and production's secrets don't reach development", %{db: db} do - {:ok, _} = Secrets.set(db, @project, "", "key", "project-wide", nil) - {:ok, _} = Secrets.set(db, @project, "development", "key", "for-development", nil) + {:ok, _} = Secrets.set(db, @project, "*", "key", "project-wide", nil) + {:ok, _} = Secrets.set(db, @project, "development/*", "key", "for-development", nil) + {:ok, _} = Secrets.set(db, @project, "development/joe/*", "key", "under-joe", nil) {:ok, _} = Secrets.set(db, @project, "development/joe", "key", "for-joe", nil) {:ok, _} = Secrets.set(db, @project, "production", "key", "for-production", nil) + # An exact scope beats a prefix, and a longer prefix beats a shorter. assert {:ok, "for-joe"} = Secrets.resolve(db, @project, "development/joe", "key") - assert {:ok, "for-joe"} = Secrets.resolve(db, @project, "development/joe/feature", "key") + assert {:ok, "under-joe"} = Secrets.resolve(db, @project, "development/joe/feature", "key") assert {:ok, "for-development"} = Secrets.resolve(db, @project, "development/sam", "key") assert {:ok, "project-wide"} = Secrets.resolve(db, @project, "staging", "key") assert {:ok, "for-production"} = Secrets.resolve(db, @project, "production", "key") + # `development/*` doesn't select `development` itself. + assert {:ok, "project-wide"} = Secrets.resolve(db, @project, "development", "key") + # A prefix is a path prefix, not a string prefix. assert {:ok, "project-wide"} = Secrets.resolve(db, @project, "developments", "key") @@ -50,8 +55,8 @@ defmodule Coflux.AdminSecretsTest do end test "a ciphertext is bound to its row", %{db: db} do - {:ok, _} = Secrets.set(db, @project, "", "a", "value-a", nil) - {:ok, _} = Secrets.set(db, @project, "", "b", "value-b", nil) + {:ok, _} = Secrets.set(db, @project, "*", "a", "value-a", nil) + {:ok, _} = Secrets.set(db, @project, "*", "b", "value-b", nil) {:ok, _} = Store.query( @@ -70,14 +75,14 @@ defmodule Coflux.AdminSecretsTest do end test "a launcher config is given the values it names", %{db: db} do - {:ok, _} = Secrets.set(db, @project, "", "k8s-token", "bearer-xyz", nil) - {:ok, _} = Secrets.set(db, @project, "", "openai", "sk-123", nil) + {:ok, _} = Secrets.set(db, @project, "*", "k8s-token", "bearer-xyz", nil) + {:ok, _} = Secrets.set(db, @project, "*", "openai", "sk-123", nil) {:ok, _} = Secrets.set( db, @project, - "", + "*", "aws", ~s({"Version": 1, "AccessKeyId": "AKIA1", "SecretAccessKey": "s3cr3t", "SessionToken": "tok"}), nil @@ -109,7 +114,7 @@ defmodule Coflux.AdminSecretsTest do assert {:error, {:secret_not_found, "nope"}} = Secrets.resolve_launcher(db, @project, "dev", missing) - {:ok, _} = Secrets.set(db, @project, "", "not-json", "plain text", nil) + {:ok, _} = Secrets.set(db, @project, "*", "not-json", "plain text", nil) assert {:error, {:secret_invalid, "not-json"}} = Secrets.resolve_launcher(db, @project, "dev", %{credentials_secret: "not-json"}) @@ -136,7 +141,7 @@ defmodule Coflux.AdminSecretsWithoutServerSecretTest do end test "without a server secret nothing can be stored or read", %{db: db} do - assert {:error, :no_secret} = Secrets.set(db, "proj", "", "x", "v", nil) + assert {:error, :no_secret} = Secrets.set(db, "proj", "*", "x", "v", nil) assert {:error, :no_secret} = Secrets.resolve(db, "proj", "dev", "x") end end diff --git a/server/test/coflux/scopes_test.exs b/server/test/coflux/scopes_test.exs index b65cdae1..3e1ed869 100644 --- a/server/test/coflux/scopes_test.exs +++ b/server/test/coflux/scopes_test.exs @@ -4,73 +4,116 @@ defmodule Coflux.ScopesTest do alias Coflux.Scopes describe "covers?/2" do - test "a scope covers itself and everything under it" do + test "an exact scope selects that workspace and no other" do assert Scopes.covers?("development", "development") - assert Scopes.covers?("development", "development/joe") - assert Scopes.covers?("development", "development/joe/feature-1") - assert Scopes.covers?("development/joe", "development/joe/feature-1") + refute Scopes.covers?("development", "development/joe") + refute Scopes.covers?("development", "development-2") end - test "the root scope covers every workspace" do - assert Scopes.covers?("", "development") - assert Scopes.covers?("", "development/joe") + test "a prefix scope selects what's under it, at any depth" do + assert Scopes.covers?("development/*", "development/joe") + assert Scopes.covers?("development/*", "development/joe/feature-1") end - test "names are compared a segment at a time" do - refute Scopes.covers?("development", "development-2") - refute Scopes.covers?("development", "development2/joe") - refute Scopes.covers?("development/joe", "development/joel") + test "a prefix scope doesn't select the workspace it names" do + refute Scopes.covers?("development/*", "development") end - test "a scope doesn't cover what's above it, or a sibling" do - refute Scopes.covers?("development/joe", "development") - refute Scopes.covers?("development", "") - refute Scopes.covers?("development/joe", "development/sam") - refute Scopes.covers?("development", "production") + test "prefixes stop at a segment boundary" do + refute Scopes.covers?("development/*", "development-2/joe") + refute Scopes.covers?("dev/*", "devops/thing") end - test "the scope of a pattern granting nothing covers nothing" do - refute Scopes.covers?(:never, "development") - refute Scopes.covers?(:never, "") + test "'*' selects every workspace" do + assert Scopes.covers?("*", "development") + assert Scopes.covers?("*", "development/joe/feature-1") + end + + test "covering a workspace and everything under it takes both scopes" do + scopes = ["development", "development/*"] + + assert Scopes.covers_any?(scopes, "development") + assert Scopes.covers_any?(scopes, "development/joe") + refute Scopes.covers_any?(scopes, "development-2") end end - describe "from_pattern/1" do - test "a bare name and a wildcard grant the same subtree" do - assert Scopes.from_pattern("development") == "development" - assert Scopes.from_pattern("development/*") == "development" - assert Scopes.from_pattern("development/joe/*") == "development/joe" + describe "contains?/2" do + test "a scope contains itself" do + assert Scopes.contains?("development", "development") + assert Scopes.contains?("development/*", "development/*") + assert Scopes.contains?("*", "*") end - test "a wildcard grant includes the workspace it names" do - scope = Scopes.from_pattern("development/*") + test "'*' contains everything, and nothing else contains '*'" do + assert Scopes.contains?("*", "development") + assert Scopes.contains?("*", "development/*") - assert Scopes.covers?(scope, "development") - assert Scopes.covers?(scope, "development/joe") + refute Scopes.contains?("development", "*") + refute Scopes.contains?("development/*", "*") end - test "'*' grants the root scope" do - assert Scopes.from_pattern("*") == "" - assert Scopes.covers?(Scopes.from_pattern("*"), "anything/at/all") + test "a prefix contains the scopes beneath it" do + assert Scopes.contains?("development/*", "development/joe") + assert Scopes.contains?("development/*", "development/joe/*") + assert Scopes.contains?("development/*", "development/joe/feature-1") end - test "a pattern that names nothing grants nothing, not everything" do - assert Scopes.from_pattern("") == :never - assert Scopes.from_pattern("/*") == :never - assert Scopes.from_pattern(nil) == :never - assert Scopes.from_pattern(["development"]) == :never + test "a prefix doesn't contain the workspace it names, which it doesn't select" do + refute Scopes.contains?("development/*", "development") + end + + test "an exact scope contains only itself" do + refute Scopes.contains?("development", "development/joe") + refute Scopes.contains?("development", "development/*") + end + + test "containment stops at a segment boundary" do + refute Scopes.contains?("dev/*", "devops/*") + refute Scopes.contains?("dev/*", "devops") + end + + # The distinction that matters: holding one workspace is not holding a + # scope that reaches others. + test "covering a workspace is not containing a scope that names it" do + assert Scopes.covers?("staging", "staging") + refute Scopes.contains?("staging", "staging/*") end end - describe "valid_pattern?/1" do - test "accepts what names a scope, rejects what doesn't" do - assert Scopes.valid_pattern?("*") - assert Scopes.valid_pattern?("development") - assert Scopes.valid_pattern?("development/*") + describe "valid?/1" do + test "accepts scopes" do + assert Scopes.valid?("*") + assert Scopes.valid?("development") + assert Scopes.valid?("development/joe") + assert Scopes.valid?("development/*") + assert Scopes.valid?("development/joe/*") + end + + test "rejects what isn't one" do + refute Scopes.valid?("") + refute Scopes.valid?("/*") + refute Scopes.valid?("development/") + refute Scopes.valid?("not valid!") + refute Scopes.valid?(nil) + refute Scopes.valid?(["development"]) + end + end + + describe "specificity/1" do + test "the nearest scope wins: exact, then longer prefix, then '*'" do + scopes = ["*", "development/*", "development/joe/*", "development/joe/feature-1"] + + assert Enum.max_by(scopes, &Scopes.specificity/1) == "development/joe/feature-1" + + assert scopes + |> Enum.reject(&(&1 == "development/joe/feature-1")) + |> Enum.max_by(&Scopes.specificity/1) == "development/joe/*" + end - refute Scopes.valid_pattern?("") - refute Scopes.valid_pattern?("/*") - refute Scopes.valid_pattern?(nil) + test "an exact scope beats a longer prefix" do + assert Scopes.specificity("development/joe") > + Scopes.specificity("development/joe/very/long/*") end end end diff --git a/tests/support/cli.py b/tests/support/cli.py index 04347063..8e85c3df 100644 --- a/tests/support/cli.py +++ b/tests/support/cli.py @@ -393,17 +393,9 @@ def pools_export(host=None, workspace="default"): return result.stdout -def _secret_scope_args(scope, global_): - if global_: - return ["--global"] - if scope is not None: - return ["--scope", scope] - return [] - - -def secrets_set(name, value, scope=None, global_=False, host=None, workspace="default"): +def secrets_set(name, value, workspaces, host=None, workspace="default"): """Set a secret, with the value on stdin as a user would give it.""" - args = ["secrets", "set", name, *_secret_scope_args(scope, global_)] + args = ["secrets", "set", name, "--workspaces", workspaces] _coflux(*args, host=host, workspace=workspace, output=None, input=value) @@ -412,8 +404,8 @@ def secrets_list(host=None, workspace="default"): return json.loads(result.stdout) -def secrets_delete(name, scope=None, global_=False, host=None, workspace="default"): - args = ["secrets", "delete", name, *_secret_scope_args(scope, global_)] +def secrets_delete(name, workspaces, host=None, workspace="default"): + args = ["secrets", "delete", name, "--workspaces", workspaces] _coflux(*args, host=host, workspace=workspace, output=None) diff --git a/tests/test_pools.py b/tests/test_pools.py index def1314e..3240d20a 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -663,7 +663,7 @@ def test_secret_env_reaches_worker(self, pool_env): ] # Set for the current workspace, which is where the pool is. - cli.secrets_set("api-key", "s3cr3t-value", host=host) + cli.secrets_set("api-key", "s3cr3t-value", workspaces="default", host=host) cli.pools_create( "secret-pool", @@ -705,7 +705,7 @@ def test_missing_secret_is_refused(self, pool_env): assert "secrets_not_found" in exc_info.value.stderr assert "nope" in exc_info.value.stderr - cli.secrets_set("k8s-token", "bearer", host=host) + cli.secrets_set("k8s-token", "bearer", workspaces="default", host=host) self._kubernetes_pool(host) with pytest.raises(subprocess.CalledProcessError) as exc_info: @@ -721,10 +721,10 @@ def test_missing_secret_is_refused(self, pool_env): assert "secrets_not_found" in exc_info.value.stderr def test_scope_follows_workspace_names(self, pool_env): - """A secret for 'development' serves 'development/joe' and not + """A secret for 'development/*' serves 'development/joe' and not 'production', whatever the workspaces inherit from.""" host = pool_env["host"] - cli.secrets_set("k8s-token", "bearer", scope="development", host=host) + cli.secrets_set("k8s-token", "bearer", workspaces="development/*", host=host) self._kubernetes_pool(host, workspace="development/joe") @@ -736,7 +736,7 @@ def test_export_names_secrets(self, pool_env, tmp_path): """An export carries the names, never the values, and imports back as long as the secrets exist.""" host = pool_env["host"] - cli.secrets_set("k8s-token", "super-secret-token", host=host) + cli.secrets_set("k8s-token", "super-secret-token", workspaces="default", host=host) self._kubernetes_pool(host) cli._coflux( "pools", @@ -816,7 +816,7 @@ def _setup_ecs_pool(ecs_env, targets, modules=None, pool_name="ecs-pool", sets=( cli.secrets_set( "aws-test", json.dumps({"AccessKeyId": "AKIATEST", "SecretAccessKey": "test-secret-key"}), - global_=True, + workspaces="*", host=host, ) diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 8741a155..545b10a6 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -11,50 +11,74 @@ class TestSecrets: def test_set_list_delete(self, server, project_id): host = f"{project_id}.localhost:{server.port}" - # The default scope is the current workspace. - cli.secrets_set("api-key", "first", host=host) + cli.secrets_set("api-key", "first", workspaces="default", host=host) [secret] = cli.secrets_list(host=host) assert secret["name"] == "api-key" assert secret["scope"] == "default" assert secret["version"] == 1 # Setting it again replaces the value and bumps the version. - cli.secrets_set("api-key", "second\n", host=host) + cli.secrets_set("api-key", "second\n", workspaces="default", host=host) [secret] = cli.secrets_list(host=host) assert secret["version"] == 2 - cli.secrets_set("shared", "x", global_=True, host=host) - cli.secrets_set("shared", "y", scope="development", host=host) + cli.secrets_set("shared", "x", workspaces="*", host=host) + cli.secrets_set("shared", "y", workspaces="development/*", host=host) listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} assert listed == { ("default", "api-key"), - ("", "shared"), - ("development", "shared"), + ("*", "shared"), + ("development/*", "shared"), } # Values are nowhere in the listing. assert "first" not in json.dumps(cli.secrets_list(host=host)) assert "second" not in json.dumps(cli.secrets_list(host=host)) - cli.secrets_delete("shared", global_=True, host=host) - cli.secrets_delete("api-key", host=host) + cli.secrets_delete("shared", workspaces="*", host=host) + cli.secrets_delete("api-key", workspaces="default", host=host) listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} - assert listed == {("development", "shared")} + assert listed == {("development/*", "shared")} with pytest.raises(subprocess.CalledProcessError): - cli.secrets_delete("api-key", host=host) + cli.secrets_delete("api-key", workspaces="default", host=host) - def test_names_and_scopes_are_validated(self, server, project_id): + def test_one_value_can_be_set_for_several_workspaces(self, server, project_id): + """Commas set the same value once per pattern: each is listed, + rotated and deleted on its own.""" + host = f"{project_id}.localhost:{server.port}" + + cli.secrets_set("api-key", "shared", workspaces="staging,production/*", host=host) + listed = {(s["scope"], s["version"]) for s in cli.secrets_list(host=host)} + assert listed == {("staging", 1), ("production/*", 1)} + + # Rotating one leaves the other where it was. + cli.secrets_set("api-key", "rotated", workspaces="staging", host=host) + listed = {(s["scope"], s["version"]) for s in cli.secrets_list(host=host)} + assert listed == {("staging", 2), ("production/*", 1)} + + # Deleting takes the patterns it was actually set for. + cli.secrets_delete("api-key", workspaces="staging,production/*", host=host) + assert cli.secrets_list(host=host) == [] + + def test_names_and_workspaces_are_validated(self, server, project_id): host = f"{project_id}.localhost:{server.port}" with pytest.raises(subprocess.CalledProcessError) as exc_info: - cli.secrets_set("not valid!", "v", host=host) + cli.secrets_set("not valid!", "v", workspaces="default", host=host) assert "bad_request" in exc_info.value.stderr with pytest.raises(subprocess.CalledProcessError) as exc_info: - cli.secrets_set("ok", "v", scope="trailing/", host=host) + cli.secrets_set("ok", "v", workspaces="trailing/", host=host) assert "bad_request" in exc_info.value.stderr + # The workspaces are required - there is no implicit scope. + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli._coflux( + "secrets", "set", "ok", host=host, workspace="default", output=None, input="v" + ) + assert "workspaces" in exc_info.value.stderr + with pytest.raises(subprocess.CalledProcessError) as exc_info: - cli.secrets_set("ok", "", host=host) + cli.secrets_set("ok", "", workspaces="default", host=host) assert "no value given" in exc_info.value.stderr diff --git a/tests/test_tokens.py b/tests/test_tokens.py index f33c9f6b..42fe8e20 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -87,64 +87,66 @@ def test_token_outlives_rotation_and_restart(self, token_server): token_server.restart(timeout=30) assert _discover(port, project_id, created["token"]) == (200, ["*"]) - def _set_secret(self, port, project_id, token, scope): - """The status of setting a secret for a scope - a convenient probe - for what a grant covers.""" + def _set_secret(self, port, project_id, token, workspaces): + """The status of setting a secret for some workspace patterns - a + convenient probe for what a grant contains.""" try: api_post( port, project_id, "set_secret", token=token, - body={"name": "key", "scope": scope, "value": "v"}, + body={"name": "key", "workspaces": workspaces, "value": "v"}, ) return 200 except urllib.error.HTTPError as e: return e.code - def test_setting_a_secret_takes_operator_access_to_its_scope(self, token_server): - """A token for 'development/*' can set a secret for a development - workspace, but not for the project or for production.""" + def test_setting_a_secret_takes_a_grant_containing_the_whole_scope(self, token_server): + """A token for 'development/*' can set a secret for that, or for + anything inside it - but not for the project, and not for a + workspace outside it.""" port = token_server.port project_id = f"tok-{uuid.uuid4().hex[:8]}" restricted = _create(port, project_id, name="dev", workspaces=["development/*"]) - def set_secret(scope): - return self._set_secret(port, project_id, restricted["token"], scope) + def set_secret(*workspaces): + return self._set_secret(port, project_id, restricted["token"], list(workspaces)) + assert set_secret("development/*") == 200 assert set_secret("development/joe") == 200 - assert set_secret("") == 403 + assert set_secret("development/joe/*") == 200 + + assert set_secret("*") == 403 assert set_secret("production") == 403 + assert set_secret("development") == 403 + + # Every pattern has to be allowed, not just one of them. + assert set_secret("development/joe", "production") == 403 - def test_a_grant_covers_the_workspace_it_names_and_everything_under_it( + def test_holding_a_workspace_is_not_holding_a_scope_that_reaches_past_it( self, token_server ): - """A pattern grants a subtree: 'development' and 'development/*' - both cover 'development' itself and everything below it. A sibling - that merely starts with the same characters isn't below it.""" + """The case that used to leak: a token for exactly 'staging' can + set a secret there, but not one reaching the workspaces beneath + it, which it has no access to.""" port = token_server.port project_id = f"tok-{uuid.uuid4().hex[:8]}" - bare = _create(port, project_id, name="bare", workspaces=["development"]) - wildcard = _create(port, project_id, name="wild", workspaces=["development/*"]) + restricted = _create(port, project_id, name="staging", workspaces=["staging"]) - for created in [bare, wildcard]: - token = created["token"] + def set_secret(*workspaces): + return self._set_secret(port, project_id, restricted["token"], list(workspaces)) - assert self._set_secret(port, project_id, token, "development") == 200 - assert self._set_secret(port, project_id, token, "development/joe") == 200 - assert ( - self._set_secret(port, project_id, token, "development/joe/feature-1") - == 200 - ) + assert set_secret("staging") == 200 - assert self._set_secret(port, project_id, token, "development-2") == 403 - assert self._set_secret(port, project_id, token, "") == 403 + assert set_secret("staging/*") == 403 + assert set_secret("staging/feature-1") == 403 def test_a_token_cannot_be_granted_more_than_its_creator_has(self, token_server): - """A token can hand on a scope within its own, and no more.""" + """A token can hand on a scope its own contains, and no more.""" port = token_server.port project_id = f"tok-{uuid.uuid4().hex[:8]}" - parent = _create(port, project_id, name="parent", workspaces=["development"]) + parent = _create(port, project_id, name="parent", workspaces=["development/*"]) def create(workspaces): try: @@ -159,13 +161,14 @@ def create(workspaces): except urllib.error.HTTPError as e: return e.code - assert create(["development"]) == 200 assert create(["development/*"]) == 200 assert create(["development/joe"]) == 200 + assert create(["development/joe/*"]) == 200 assert create(["*"]) == 403 + assert create(["development"]) == 403 assert create(["production"]) == 403 - assert create(["development-2"]) == 403 + assert create(["development-2/*"]) == 403 # A pattern that names no scope is rejected outright. assert create([""]) == 400 From 20c97c4ea9bcc4622e4cd2c405996e88ea5bd457 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 11:18:42 +0100 Subject: [PATCH 10/21] Rename field for consistency --- cli/cmd/coflux/secrets.go | 4 ++-- server/lib/coflux/admin/secrets.ex | 16 +++++++------- server/lib/coflux/auth.ex | 22 +++++++++---------- server/lib/coflux/events.ex | 4 ++-- server/lib/coflux/handlers/api.ex | 11 ++++------ server/lib/coflux/orchestration/server.ex | 17 ++++++-------- .../orchestration/server/permissions.ex | 2 -- .../coflux/orchestration/server/snapshots.ex | 2 +- server/lib/coflux/topics/secrets.ex | 16 +++++++------- server/priv/migrations/admin/2.sql | 14 +++++++----- server/test/coflux/admin_secrets_test.exs | 2 +- tests/test_secrets.py | 10 ++++----- 12 files changed, 56 insertions(+), 64 deletions(-) diff --git a/cli/cmd/coflux/secrets.go b/cli/cmd/coflux/secrets.go index f870233d..f3470704 100644 --- a/cli/cmd/coflux/secrets.go +++ b/cli/cmd/coflux/secrets.go @@ -175,7 +175,7 @@ func runSecretsList(cmd *cobra.Command, args []string) error { } sort.Slice(secrets, func(i, j int) bool { - si, sj := getString(secrets[i], "scope"), getString(secrets[j], "scope") + si, sj := getString(secrets[i], "workspaces"), getString(secrets[j], "workspaces") if si != sj { return si < sj } @@ -199,7 +199,7 @@ func runSecretsList(cmd *cobra.Command, args []string) error { } rows = append(rows, []string{ getString(s, "name"), - getString(s, "scope"), + getString(s, "workspaces"), fmt.Sprintf("%d", int(getFloat64(s, "version"))), formatTimestamp(getInt64(s, "updatedAt")), by, diff --git a/server/lib/coflux/admin/secrets.ex b/server/lib/coflux/admin/secrets.ex index d133b87d..c4b73ba1 100644 --- a/server/lib/coflux/admin/secrets.ex +++ b/server/lib/coflux/admin/secrets.ex @@ -50,7 +50,7 @@ defmodule Coflux.Admin.Secrets do {existing_id, created_at, version} = case Store.query_one( db, - "SELECT id, created_at, version FROM secrets WHERE scope = ?1 AND name = ?2", + "SELECT id, created_at, version FROM secrets WHERE workspaces = ?1 AND name = ?2", {scope, name} ) do {:ok, {id, created_at, version}} -> {id, created_at, version + 1} @@ -89,7 +89,7 @@ defmodule Coflux.Admin.Secrets do else {:ok, _} = Store.insert_one(db, :secrets, %{ - scope: scope, + workspaces: scope, name: name, version: version, key_id: @key_id, @@ -104,7 +104,7 @@ defmodule Coflux.Admin.Secrets do {:ok, %{ - scope: scope, + workspaces: scope, name: name, version: version, created_at: created_at, @@ -117,7 +117,7 @@ defmodule Coflux.Admin.Secrets do def delete(db, scope, name) do case Store.query_one( db, - "SELECT id FROM secrets WHERE scope = ?1 AND name = ?2", + "SELECT id FROM secrets WHERE workspaces = ?1 AND name = ?2", {scope, name} ) do {:ok, {id}} -> @@ -135,9 +135,9 @@ defmodule Coflux.Admin.Secrets do Store.query( db, """ - SELECT scope, name, version, created_at, updated_at, updated_by_type, updated_by_external_id + SELECT workspaces, name, version, created_at, updated_at, updated_by_type, updated_by_external_id FROM secrets - ORDER BY scope, name + ORDER BY workspaces, name """, {} ) @@ -145,7 +145,7 @@ defmodule Coflux.Admin.Secrets do {:ok, Enum.map(rows, fn {scope, name, version, created_at, updated_at, type, external_id} -> %{ - scope: scope, + workspaces: scope, name: name, version: version, created_at: created_at, @@ -302,7 +302,7 @@ defmodule Coflux.Admin.Secrets do {:ok, rows} = Store.query( db, - "SELECT scope, version, nonce, ciphertext FROM secrets WHERE name = ?1", + "SELECT workspaces, version, nonce, ciphertext FROM secrets WHERE name = ?1", {name} ) diff --git a/server/lib/coflux/auth.ex b/server/lib/coflux/auth.ex index 41a373ed..4574fe35 100644 --- a/server/lib/coflux/auth.ex +++ b/server/lib/coflux/auth.ex @@ -51,15 +51,15 @@ defmodule Coflux.Auth do A grant is a list of workspace patterns, which control write access. `Coflux.Scopes` defines what a pattern grants, and `Coflux.Orchestration.Server.Permissions` applies it - the patterns are - carried through this module unread, beyond normalising a grant that - includes "*" to `:all`. + carried through this module unread, beyond collapsing a grant that + includes "*" to just that. """ alias Coflux.{Config, JwksStore, Orchestration} @type access :: %{ type: :super | :none | :service | :studio | :session, - workspaces: :all | [String.t()], + workspaces: [String.t()], principal_id: integer() | nil } @@ -73,7 +73,7 @@ defmodule Coflux.Auth do Returns `{:ok, access}` with access details when allowed. The access map contains: - - workspaces: :all | [String.t()] - workspace patterns the token has access to + - workspaces: [String.t()] - workspace patterns the token has access to - principal_id: integer() | nil - the database ID of the principal (nil for super token or anonymous) Returns `{:error, :unauthorized}` otherwise. @@ -87,7 +87,7 @@ defmodule Coflux.Auth do if Config.require_auth?() do {:error, :unauthorized} else - {:ok, %{type: :none, workspaces: :all, principal_id: nil}} + {:ok, %{type: :none, workspaces: ["*"], principal_id: nil}} end end @@ -189,7 +189,7 @@ defmodule Coflux.Auth do expected_hash -> if hash_token(token) == expected_hash do # Super token has full access, no principal - {:ok, %{type: :super, workspaces: :all, principal_id: nil}} + {:ok, %{type: :super, workspaces: ["*"], principal_id: nil}} else :error end @@ -379,13 +379,11 @@ defmodule Coflux.Auth do end end + # Unrestricted access is the `*` pattern, not a separate shape - so + # every grant is a list of patterns and there is one thing to match. defp normalize_workspaces(workspaces) when is_list(workspaces) do - if "*" in workspaces do - :all - else - workspaces - end + if "*" in workspaces, do: ["*"], else: workspaces end - defp normalize_workspaces(_), do: :all + defp normalize_workspaces(_), do: ["*"] end diff --git a/server/lib/coflux/events.ex b/server/lib/coflux/events.ex index eb4b7aae..d8487dcc 100644 --- a/server/lib/coflux/events.ex +++ b/server/lib/coflux/events.ex @@ -321,12 +321,12 @@ defmodule Coflux.Events.SecretSet do Row: `secrets` (admin store), created or replaced. Its value is not an event: nothing that carries this ever sees it. """ - defstruct [:scope, :name, :version, :created_at, :updated_at, :updated_by] + defstruct [:workspaces, :name, :version, :created_at, :updated_at, :updated_by] end defmodule Coflux.Events.SecretDeleted do @moduledoc "Row: `secrets` (admin store), gone." - defstruct [:scope, :name] + defstruct [:workspaces, :name] end defmodule Coflux.Events.WorkspaceCreated do diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 79e8d998..8c688d91 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -76,16 +76,13 @@ defmodule Coflux.Handlers.Api do # Whether the caller can grant the requested access: each scope asked for # has to be contained whole by one the caller holds. Holding a workspace # inside a scope is not holding the scope. - defp workspaces_covered?(:all, _requested), do: true defp workspaces_covered?(_caller, nil), do: true defp workspaces_covered?(caller_scopes, requested) do Enum.all?(requested, &Scopes.contains_any?(caller_scopes, &1)) end - defp handle(req, "GET", ["discover"], _project_id, %{workspaces: workspaces}) do - patterns = if workspaces == :all, do: ["*"], else: workspaces - + defp handle(req, "GET", ["discover"], _project_id, %{workspaces: patterns}) do json_response(req, %{ "version" => Version.version(), "api_version" => Version.api_version(), @@ -948,7 +945,7 @@ defmodule Coflux.Handlers.Api do # If no workspaces specified, inherit caller's workspaces (unless caller has full access) effective_workspaces = case {requested_workspaces, access.workspaces} do - {nil, :all} -> nil + {nil, ["*"]} -> nil {nil, patterns} -> patterns {requested, _} -> requested end @@ -995,7 +992,7 @@ defmodule Coflux.Handlers.Api do "name" => arguments.name, "secrets" => Enum.map(secrets, fn secret -> - %{"workspaces" => secret.scope, "version" => secret.version} + %{"workspaces" => secret.workspaces, "version" => secret.version} end) }) @@ -1046,7 +1043,7 @@ defmodule Coflux.Handlers.Api do {:ok, token} -> # Allow revocation if caller has full access OR created this token can_revoke = - access.workspaces == :all or + "*" in access.workspaces or token.created_by_principal_id == access[:principal_id] if can_revoke do diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index ae5c45ef..0415c347 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -339,7 +339,7 @@ defmodule Coflux.Orchestration.Server do {:ok, secret} -> state = Effects.emit(state, %SecretSet{ - scope: secret.scope, + workspaces: secret.workspaces, name: secret.name, version: secret.version, created_at: secret.created_at, @@ -366,7 +366,8 @@ defmodule Coflux.Orchestration.Server do Enum.reduce(scopes, {state, []}, fn scope, {state, deleted} -> case Coflux.Admin.Secrets.delete(state.admin_db, scope, name) do :ok -> - {Effects.emit(state, %SecretDeleted{scope: scope, name: name}), deleted ++ [scope]} + {Effects.emit(state, %SecretDeleted{workspaces: scope, name: name}), + deleted ++ [scope]} {:error, :not_found} -> {state, deleted} @@ -3012,15 +3013,11 @@ defmodule Coflux.Orchestration.Server do defp check_secret_scope_access(nil, _scopes), do: :ok defp check_secret_scope_access(access, scopes) do - case access[:workspaces] do - :all -> - :ok + granted = access[:workspaces] - granted -> - if Enum.all?(scopes, &Scopes.contains_any?(granted, &1)), - do: :ok, - else: {:error, :forbidden} - end + if Enum.all?(scopes, &Scopes.contains_any?(granted, &1)), + do: :ok, + else: {:error, :forbidden} end defp principal_identity(state, access) do diff --git a/server/lib/coflux/orchestration/server/permissions.ex b/server/lib/coflux/orchestration/server/permissions.ex index 7ebe66e0..41d420ce 100644 --- a/server/lib/coflux/orchestration/server/permissions.ex +++ b/server/lib/coflux/orchestration/server/permissions.ex @@ -60,8 +60,6 @@ defmodule Coflux.Orchestration.Server.Permissions do end end - def operator?(:all, _workspace), do: true - def operator?(scopes, workspace), do: Scopes.covers_any?(scopes, workspace) def check_operator_access(nil, _name), do: :ok diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index a8d1cbe9..bcac0a40 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -214,7 +214,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do {:ok, Enum.map(secrets, fn secret -> %SecretSet{ - scope: secret.scope, + workspaces: secret.workspaces, name: secret.name, version: secret.version, created_at: secret.created_at, diff --git a/server/lib/coflux/topics/secrets.ex b/server/lib/coflux/topics/secrets.ex index 49bde170..dbbe9eef 100644 --- a/server/lib/coflux/topics/secrets.ex +++ b/server/lib/coflux/topics/secrets.ex @@ -1,7 +1,7 @@ defmodule Coflux.Topics.Secrets do @moduledoc """ - The project's secrets - what exists, in which scope, and when it last - changed. Never a value. + The project's secrets - what exists, for which workspaces, and when it + last changed. Never a value. """ use Topical.Topic, route: ["secrets"] @@ -51,10 +51,10 @@ defmodule Coflux.Topics.Secrets.Model do end def apply(model, %SecretSet{} = e) do - key = key(e.scope, e.name) + key = key(e.workspaces, e.name) secret = %{ - scope: e.scope, + workspaces: e.workspaces, name: e.name, version: e.version, created_at: e.created_at, @@ -66,12 +66,12 @@ defmodule Coflux.Topics.Secrets.Model do end def apply(model, %SecretDeleted{} = e) do - key = key(e.scope, e.name) + key = key(e.workspaces, e.name) {Map.delete(model, key), [key]} end - # A name is unique within a scope, and a scope can be empty. - defp key(scope, name), do: "#{name}@#{scope}" + # A name is unique within a workspace pattern. + defp key(workspaces, name), do: "#{name}@#{workspaces}" def project(model), do: Map.new(model, fn {key, _} -> {key, project_entry(model, key)} end) @@ -79,7 +79,7 @@ defmodule Coflux.Topics.Secrets.Model do case Map.fetch(model, key) do {:ok, secret} -> %{ - scope: secret.scope, + workspaces: secret.workspaces, name: secret.name, version: secret.version, createdAt: secret.created_at, diff --git a/server/priv/migrations/admin/2.sql b/server/priv/migrations/admin/2.sql index 814c546b..bfc3a248 100644 --- a/server/priv/migrations/admin/2.sql +++ b/server/priv/migrations/admin/2.sql @@ -1,15 +1,17 @@ -- Secrets: values a pool needs that must never appear in its -- configuration. Encrypted with a key derived from COFLUX_SECRET and bound --- to their row (project, scope, name, version), so a ciphertext moved --- elsewhere won't decrypt. One current value per scope and name: setting +-- to their row (project, workspaces, name, version), so a ciphertext moved +-- elsewhere won't decrypt. One current value per pattern and name: setting -- it again replaces the value and bumps the version, deleting it removes -- it, and nothing older is kept. -- --- `scope` is a workspace name, or a prefix of one ('development' covers --- 'development/joe'); '' is the whole project. +-- `workspaces` is one workspace pattern - 'development', 'development/*' +-- or '*'. A secret set for several is stored once per pattern, so each is +-- versioned and deleted on its own. (Contrast `tokens.workspaces`, which +-- is a JSON array: a token holds its patterns together.) CREATE TABLE secrets ( id INTEGER PRIMARY KEY, - scope TEXT NOT NULL, + workspaces TEXT NOT NULL, name TEXT NOT NULL, version INTEGER NOT NULL, key_id TEXT NOT NULL, @@ -19,6 +21,6 @@ CREATE TABLE secrets ( updated_at INTEGER NOT NULL, updated_by_type TEXT, -- 'user' or 'token' updated_by_external_id TEXT, - UNIQUE (scope, name), + UNIQUE (workspaces, name), CHECK ((updated_by_type IS NULL) = (updated_by_external_id IS NULL)) ) STRICT; diff --git a/server/test/coflux/admin_secrets_test.exs b/server/test/coflux/admin_secrets_test.exs index 50cfc7f1..880b2631 100644 --- a/server/test/coflux/admin_secrets_test.exs +++ b/server/test/coflux/admin_secrets_test.exs @@ -24,7 +24,7 @@ defmodule Coflux.AdminSecretsTest do assert {:ok, "second"} = Secrets.resolve(db, @project, "development", "api-key") - assert {:ok, [%{name: "api-key", scope: "*", version: 2}]} = Secrets.list(db) + assert {:ok, [%{name: "api-key", workspaces: "*", version: 2}]} = Secrets.list(db) assert :ok = Secrets.delete(db, "*", "api-key") assert {:error, :not_found} = Secrets.delete(db, "*", "api-key") assert {:error, :not_found} = Secrets.resolve(db, @project, "development", "api-key") diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 545b10a6..c902e012 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -14,7 +14,7 @@ def test_set_list_delete(self, server, project_id): cli.secrets_set("api-key", "first", workspaces="default", host=host) [secret] = cli.secrets_list(host=host) assert secret["name"] == "api-key" - assert secret["scope"] == "default" + assert secret["workspaces"] == "default" assert secret["version"] == 1 # Setting it again replaces the value and bumps the version. @@ -24,7 +24,7 @@ def test_set_list_delete(self, server, project_id): cli.secrets_set("shared", "x", workspaces="*", host=host) cli.secrets_set("shared", "y", workspaces="development/*", host=host) - listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} + listed = {(s["workspaces"], s["name"]) for s in cli.secrets_list(host=host)} assert listed == { ("default", "api-key"), ("*", "shared"), @@ -37,7 +37,7 @@ def test_set_list_delete(self, server, project_id): cli.secrets_delete("shared", workspaces="*", host=host) cli.secrets_delete("api-key", workspaces="default", host=host) - listed = {(s["scope"], s["name"]) for s in cli.secrets_list(host=host)} + listed = {(s["workspaces"], s["name"]) for s in cli.secrets_list(host=host)} assert listed == {("development/*", "shared")} with pytest.raises(subprocess.CalledProcessError): @@ -49,12 +49,12 @@ def test_one_value_can_be_set_for_several_workspaces(self, server, project_id): host = f"{project_id}.localhost:{server.port}" cli.secrets_set("api-key", "shared", workspaces="staging,production/*", host=host) - listed = {(s["scope"], s["version"]) for s in cli.secrets_list(host=host)} + listed = {(s["workspaces"], s["version"]) for s in cli.secrets_list(host=host)} assert listed == {("staging", 1), ("production/*", 1)} # Rotating one leaves the other where it was. cli.secrets_set("api-key", "rotated", workspaces="staging", host=host) - listed = {(s["scope"], s["version"]) for s in cli.secrets_list(host=host)} + listed = {(s["workspaces"], s["version"]) for s in cli.secrets_list(host=host)} assert listed == {("staging", 2), ("production/*", 1)} # Deleting takes the patterns it was actually set for. From a6da67032a376856dccc2bc312b3c52c361b234e Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 11:19:15 +0100 Subject: [PATCH 11/21] Validate merged pool --- server/lib/coflux/orchestration/server.ex | 8 +++- server/lib/coflux/orchestration/workspaces.ex | 47 ++++++++++++++----- tests/test_pools.py | 37 +++++++++++++++ 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index 0415c347..b39076b1 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -725,7 +725,9 @@ defmodule Coflux.Orchestration.Server do ) do with {:ok, workspace_id, _} <- Permissions.require_workspace(state, workspace_external_id, access), - :ok <- check_secret_references(state, workspace_id, pool_patch[:launcher]) do + {:ok, merged} <- + Workspaces.resolve_pool_patch(state.db, workspace_id, pool_name, pool_patch), + :ok <- check_secret_references(state, workspace_id, merged[:launcher]) do case Workspaces.update_pool( state.db, workspace_id, @@ -3028,7 +3030,9 @@ defmodule Coflux.Orchestration.Server do end # A pool naming a secret its workspace can't see would never launch, so - # it is refused now rather than found out then. + # it is refused now rather than found out then. Checked against the + # definition being stored, never against a patch: a patch that names no + # secret can still leave the pool referring to one that has since gone. defp check_secret_references(state, workspace_id, launcher) when is_map(launcher) do workspace_name = state.workspaces[workspace_id].name Coflux.Admin.Secrets.check_references(state.admin_db, workspace_name, launcher) diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index 8581902e..8ec96f09 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -466,23 +466,48 @@ defmodule Coflux.Orchestration.Workspaces do # unchanged. When pool is nil, the pool is deleted. # Returns {:error, :not_found} if the pool doesn't exist. # Returns {:error, :type_change} if the patch tries to change the launcher type. + @doc """ + The definition a patch would produce, without writing it, so a caller can + check the result rather than the patch - a patch naming no secrets can + still leave a pool referring to one. `nil` when the patch would delete + the pool. + """ + def resolve_pool_patch(db, workspace_id, pool_name, pool_patch) do + with {:ok, {_pool_id, _definition_id, existing}} <- + load_pool_for_patch(db, workspace_id, pool_name), + :ok <- check_launcher_type_change(existing, pool_patch) do + {:ok, pool_patch && apply_pool_patch(existing, pool_patch)} + end + end + + # The pool's current definition, with the ids needed to replace it. + defp load_pool_for_patch(db, workspace_id, pool_name) do + case get_latest_pool(db, workspace_id, pool_name) do + {:ok, nil} -> + {:error, :not_found} + + {:ok, {pool_id, definition_id}} -> + existing = + if definition_id do + {:ok, definition} = get_pool_definition(db, definition_id) + definition + else + %{modules: [], provides: %{}, accepts: %{}, launcher: nil} + end + + {:ok, {pool_id, definition_id, existing}} + end + end + def update_pool(db, workspace_id, pool_name, pool_patch, created_by \\ nil) do with_transaction(db, fn -> now = current_timestamp() - case get_latest_pool(db, workspace_id, pool_name) do - {:ok, nil} -> + case load_pool_for_patch(db, workspace_id, pool_name) do + {:error, :not_found} -> {:error, :not_found} - {:ok, {existing_pool_id, existing_pool_definition_id}} -> - existing = - if existing_pool_definition_id do - {:ok, def} = get_pool_definition(db, existing_pool_definition_id) - def - else - %{modules: [], provides: %{}, accepts: %{}, launcher: nil} - end - + {:ok, {existing_pool_id, existing_pool_definition_id, existing}} -> # Check for type change in launcher patch with :ok <- check_launcher_type_change(existing, pool_patch) do pool = diff --git a/tests/test_pools.py b/tests/test_pools.py index 3240d20a..f5fb03cb 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -720,6 +720,43 @@ def test_missing_secret_is_refused(self, pool_env): ) assert "secrets_not_found" in exc_info.value.stderr + def test_a_patch_is_checked_against_the_pool_it_would_leave(self, pool_env): + """A patch naming no secret can still leave the pool referring to + one that has gone, so the check is against the merged definition, + not the patch.""" + host = pool_env["host"] + cli.secrets_set("k8s-token", "bearer", workspaces="default", host=host) + self._kubernetes_pool(host) + + cli.secrets_delete("k8s-token", workspaces="default", host=host) + + # Touching an unrelated field would leave `tokenSecret` dangling. + with pytest.raises(subprocess.CalledProcessError) as exc_info: + cli._coflux( + "pools", + "update", + "k8s-pool", + "--set", + "namespace=other", + host=host, + output=None, + ) + assert "secrets_not_found" in exc_info.value.stderr + assert "k8s-token" in exc_info.value.stderr + + # Removing the dangling reference in the same patch is allowed. + cli._coflux( + "pools", + "update", + "k8s-pool", + "--set", + "namespace=other", + "--unset", + "tokenSecret", + host=host, + output=None, + ) + def test_scope_follows_workspace_names(self, pool_env): """A secret for 'development/*' serves 'development/joe' and not 'production', whatever the workspaces inherit from.""" From 54e141d5f26164ea3f599f2c247744ce1f74157f Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 12:33:31 +0100 Subject: [PATCH 12/21] Support configuring ECS pool with role --- cli/CHANGELOG.md | 2 +- cli/cmd/coflux/pools.go | 11 +- docs/docs/pools.md | 29 +- server/CHANGELOG.md | 4 + server/lib/coflux/application.ex | 1 + server/lib/coflux/handlers/api.ex | 28 ++ .../lib/coflux/launchers/aws_credentials.ex | 309 +++++++++++++-- .../coflux/launchers/aws_credentials/cache.ex | 51 +++ server/lib/coflux/launchers/ecs.ex | 15 +- server/lib/coflux/topics/pool.ex | 2 + .../coflux/launchers/aws_credentials_test.exs | 370 ++++++++++++++++++ 11 files changed, 780 insertions(+), 42 deletions(-) create mode 100644 server/lib/coflux/launchers/aws_credentials/cache.ex create mode 100644 server/test/coflux/launchers/aws_credentials_test.exs diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 3b619733..e2b4b26f 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -2,7 +2,7 @@ Enhancements: -- Adds `--type ecs` support for `pools create` and `pools update`. +- Adds `--type ecs` support for `pools create` and `pools update`, including the `roleArn` and `roleExternalId` fields for a role to assume before calling ECS. - Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it. - Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. Each secret is set for one or more workspace patterns, given as a required `--workspaces`, in the same language `tokens create --workspaces` uses. diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 0ba79258..00300671 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -226,6 +226,12 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { if secret := getString(launcher, "credentialsSecret"); secret != "" { fmt.Printf("Credentials secret: %s\n", secret) } + if roleArn := getString(launcher, "roleArn"); roleArn != "" { + fmt.Printf("Role ARN: %s\n", roleArn) + } + if externalID := getString(launcher, "roleExternalId"); externalID != "" { + fmt.Printf("Role external ID: %s\n", externalID) + } if endpoint := getString(launcher, "endpoint"); endpoint != "" { fmt.Printf("Endpoint: %s\n", endpoint) } @@ -645,7 +651,8 @@ var launcherFields = map[string]bool{ "cluster": true, "taskDefinition": true, "region": true, "containerName": true, "launchType": true, "capacityProvider": true, "subnets": true, "securityGroups": true, "assignPublicIp": true, - "platformVersion": true, "credentialsSecret": true, "endpoint": true, + "platformVersion": true, "credentialsSecret": true, "roleArn": true, + "roleExternalId": true, "endpoint": true, "serverHost": true, "serverSecure": true, "adapter": true, "concurrency": true, "env": true, "envSecrets": true, } @@ -1424,6 +1431,8 @@ var camelToSnake = map[string]string{ "platformVersion": "platform_version", "tokenSecret": "token_secret", "credentialsSecret": "credentials_secret", + "roleArn": "role_arn", + "roleExternalId": "role_external_id", "envSecrets": "env_secrets", } diff --git a/docs/docs/pools.md b/docs/docs/pools.md index eea454f1..9c68c11f 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -142,11 +142,28 @@ coflux pools update mypool --set credentialsSecret=aws-sandbox ``` Without one, credentials come from the server's surroundings the way the -AWS SDKs look: `AWS_ACCESS_KEY_ID` and friends in its environment, its ECS -task role, or its EC2 instance profile. Either way they need `ecs:RunTask`, -`ecs:DescribeTasks` and `ecs:StopTask` on the cluster, -`ecs:DescribeTaskDefinition` unless `containerName` is set, and -`iam:PassRole` for the roles the task definition names. +AWS SDKs look: `AWS_ACCESS_KEY_ID` and friends in its environment, a web +identity token (`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`, as EKS +sets for a pod whose service account has a role), its ECS task role, or +its EC2 instance profile. + +Either way, `roleArn` names a role to assume with them before ECS is +called, with `roleExternalId` if the role's trust policy asks for one - so a +server in one account can launch into another, or a pool can be held to a +role that reaches only its cluster: + +```bash +coflux pools update mypool \ + --set roleArn=arn:aws:iam::123456789012:role/coflux-launcher \ + --set roleExternalId=coflux-production +``` + +Whichever identity calls ECS - the role, or the credentials themselves +when there's no role - needs `ecs:RunTask`, `ecs:DescribeTasks` and +`ecs:StopTask` on the cluster, `ecs:DescribeTaskDefinition` unless +`containerName` is set, and `iam:PassRole` for the roles the task +definition names. Credentials that assume a role need `sts:AssumeRole` on +it, and the role's trust policy has to allow them to. ECS doesn't expose container output through its API, so a worker's log tail isn't shown; a task that fails to start reports its reason in its @@ -166,6 +183,8 @@ see what workers print. | `assignPublicIp` | Give the task a public IP | | `platformVersion` | Fargate platform version | | `credentialsSecret` | Name of the secret holding AWS credentials as JSON (`AccessKeyId`, `SecretAccessKey`, optional `SessionToken`) | +| `roleArn` | IAM role to assume before calling ECS | +| `roleExternalId` | External ID the role's trust policy expects, if any | | `endpoint` | ECS API endpoint override (e.g. a VPC endpoint) | ### Common fields diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 6ae0a6be..351c46e0 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -1,5 +1,9 @@ ## 0.13.0 +Enhancements: + +- The ECS launcher can assume an IAM role (`roleArn`, with `roleExternalId` where the trust policy asks for one) with whatever credentials it finds, so a server in one account can launch into another. The search of the server's surroundings now includes a web identity token (`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`, as EKS sets for a pod whose service account has a role). Credentials issued by STS are cached until shortly before they expire. + Changes: - Workspace patterns mean one thing everywhere now. A pattern selects a workspace (`development`), the workspaces under it (`development/*`, at any depth, but not `development` itself), or all of them (`*`) — the rule tokens already used, now used for a secret's workspaces too, where a bare name previously selected everything beneath it as well. Patterns that name nothing are rejected. diff --git a/server/lib/coflux/application.ex b/server/lib/coflux/application.ex index 9c523649..60807e5b 100644 --- a/server/lib/coflux/application.ex +++ b/server/lib/coflux/application.ex @@ -12,6 +12,7 @@ defmodule Coflux.Application do children = [ # TODO: separate launch supervisor per project? (and specify max_children?) {Task.Supervisor, name: Coflux.LauncherSupervisor}, + Coflux.Launchers.AwsCredentials.Cache, {DynamicSupervisor, name: Coflux.ProcessLauncher.Supervisor, strategy: :one_for_one}, # Keyed by OS pid, so a launched worker is found by something that # means the same thing after a restart as before one. diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 8c688d91..115df419 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -8,6 +8,13 @@ defmodule Coflux.Handlers.Api do @ecs_launch_types ["FARGATE", "EC2", "EXTERNAL"] + # An IAM role ARN: any partition, a 12-digit account, and a name that + # may sit under a path. + @iam_role_arn_regex ~r/^arn:aws[a-z-]*:iam::\d{12}:role\/[\w+=,.@\/-]+$/ + + # STS's own rule for an external ID. + @external_id_regex ~r/^[\w+=,.@:\/-]{2,1224}$/ + # A directory upload arrives as one entry per file, so this bounds an # accidental drop of a very large tree. Unlike the sizes, which the # client asserts, the count is something the server can see for itself. @@ -1399,6 +1406,8 @@ defmodule Coflux.Handlers.Api do assign_public_ip = Map.get(value, "assignPublicIp") platform_version = Map.get(value, "platformVersion") credentials_secret = Map.get(value, "credentialsSecret") + role_arn = Map.get(value, "roleArn") + role_external_id = Map.get(value, "roleExternalId") endpoint = Map.get(value, "endpoint") cond do @@ -1444,6 +1453,17 @@ defmodule Coflux.Handlers.Api do not Coflux.Admin.Secrets.valid_name?(credentials_secret) -> {:error, :invalid} + not is_nil(role_arn) and + (not is_binary(role_arn) or String.length(role_arn) > 2048 or + not Regex.match?(@iam_role_arn_regex, role_arn)) -> + {:error, :invalid} + + # An external ID is something to assume a role with, so it needs one. + not is_nil(role_external_id) and + (is_nil(role_arn) or not is_binary(role_external_id) or + not Regex.match?(@external_id_regex, role_external_id)) -> + {:error, :invalid} + not is_nil(endpoint) and (not is_binary(endpoint) or String.length(endpoint) > 500 or not String.starts_with?(endpoint, ["http://", "https://"])) -> @@ -1460,6 +1480,8 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value(:assign_public_ip, if(assign_public_ip == true, do: true)) |> maybe_put_value(:platform_version, platform_version) |> maybe_put_value(:credentials_secret, credentials_secret) + |> maybe_put_value(:role_arn, role_arn) + |> maybe_put_value(:role_external_id, role_external_id) |> maybe_put_value(:endpoint, endpoint) {:ok, launcher} @@ -1651,6 +1673,8 @@ defmodule Coflux.Handlers.Api do |> maybe_put_value("assignPublicIp", Map.get(launcher, :assign_public_ip)) |> maybe_put_value("platformVersion", Map.get(launcher, :platform_version)) |> maybe_put_value("credentialsSecret", Map.get(launcher, :credentials_secret)) + |> maybe_put_value("roleArn", Map.get(launcher, :role_arn)) + |> maybe_put_value("roleExternalId", Map.get(launcher, :role_external_id)) |> maybe_put_value("endpoint", Map.get(launcher, :endpoint)) :kubernetes -> @@ -1838,6 +1862,8 @@ defmodule Coflux.Handlers.Api do {"assignPublicIp", &is_boolean/1}, {"platformVersion", &is_binary/1}, {"credentialsSecret", &Coflux.Admin.Secrets.valid_name?/1}, + {"roleArn", &(is_binary(&1) and Regex.match?(@iam_role_arn_regex, &1))}, + {"roleExternalId", &(is_binary(&1) and Regex.match?(@external_id_regex, &1))}, {"endpoint", &is_binary/1}, {"serverHost", &is_binary/1}, {"serverSecure", &is_boolean/1}, @@ -1890,6 +1916,8 @@ defmodule Coflux.Handlers.Api do "assignPublicIp" => :assign_public_ip, "platformVersion" => :platform_version, "credentialsSecret" => :credentials_secret, + "roleArn" => :role_arn, + "roleExternalId" => :role_external_id, "endpoint" => :endpoint, "serverHost" => :server_host, "serverSecure" => :server_secure, diff --git a/server/lib/coflux/launchers/aws_credentials.ex b/server/lib/coflux/launchers/aws_credentials.ex index d85ed0df..3636bb90 100644 --- a/server/lib/coflux/launchers/aws_credentials.ex +++ b/server/lib/coflux/launchers/aws_credentials.ex @@ -4,57 +4,121 @@ defmodule Coflux.Launchers.AwsCredentials do Credentials configured on the pool win. Otherwise the server's own surroundings are searched the way the AWS SDKs do, in the same order: - environment variables, then the container credentials endpoint (an ECS - task role, when the server itself runs on ECS), then the EC2 instance - metadata service (an instance profile). Nothing is cached: the - endpoints are link-local and quick, and a launcher call is rare enough - that a lookup per call costs less than getting expiry right. + environment variables, then a web identity token (the service account + token Kubernetes projects into a pod, when the server runs on EKS with + a role for its service account), then the container credentials + endpoint (an ECS task role, when the server itself runs on ECS), then + the EC2 instance metadata service (an instance profile). + + Whatever is found, a pool can name a role to assume with it, so one + server can launch into accounts its own identity doesn't reach. + + Credentials that STS issues - for an assumed role, or a web identity - + are cached until shortly before they expire: a poll runs every few + seconds per worker, and STS is neither local nor unthrottled. The rest + are looked up on every call: those endpoints are link-local and quick, + and getting their expiry right would cost more than the lookup. """ + alias Coflux.Launchers.AwsCredentials.Cache + @container_credentials_host "http://169.254.170.2" @instance_metadata_endpoint "http://169.254.169.254" @instance_metadata_token_ttl "21600" - # These endpoints are on the local link, or not there at all: a slow + @sts_version "2011-06-15" + @role_session_name "coflux" + + # Cached STS credentials are replaced this long before they expire, so + # a call made with them has time to finish. + @refresh_margin_seconds 300 + + # The local endpoints are on the link, or not there at all: a slow # answer means the latter, and a launcher task shouldn't sit on it. @connect_timeout_ms 1_000 @receive_timeout_ms 2_000 + # STS is a real service, reached over the internet. + @sts_connect_timeout_ms 5_000 + @sts_receive_timeout_ms 10_000 + @type t :: %{ - access_key_id: String.t(), - secret_access_key: String.t(), - session_token: String.t() | nil + required(:access_key_id) => String.t(), + required(:secret_access_key) => String.t(), + required(:session_token) => String.t() | nil, + optional(:expires_at) => integer() } + @type error :: :credentials_missing | {:assume_role, String.t(), String.t() | nil} + @doc """ Resolves credentials, preferring `static` (a map with `:access_key_id`, `:secret_access_key` and optionally `:session_token`) when given. + + Options: + + * `:region` - whose STS endpoint to use, when a role is assumed. + * `:role_arn` - a role to assume with the credentials found. + * `:external_id` - what the role's trust policy expects, if anything. + * `:req_options` - extra options for `Req.request/1`, so a test can + stand in for the network. + + Fails with `:credentials_missing` when nothing is found, or + `{:assume_role, code, message}` when STS refuses (the code is its own, + `AccessDenied` say) or can't be reached (`request_failed`). """ - @spec resolve(map() | nil) :: {:ok, t()} | {:error, :credentials_missing} - def resolve(%{access_key_id: access_key_id, secret_access_key: secret_access_key} = static) - when is_binary(access_key_id) and is_binary(secret_access_key) do - {:ok, - %{ - access_key_id: access_key_id, - secret_access_key: secret_access_key, - session_token: Map.get(static, :session_token) - }} - end - - def resolve(_static) do - Enum.find_value( - [&from_environment/0, &from_container/0, &from_instance_metadata/0], - {:error, :credentials_missing}, - fn source -> - case source.() do - {:ok, credentials} -> {:ok, credentials} - :none -> nil - end + @spec resolve(map() | nil, keyword()) :: {:ok, t()} | {:error, error()} + def resolve(static, opts \\ []) do + static = static_credentials(static) + + case Keyword.get(opts, :role_arn) do + nil -> + base_credentials(static, opts) + + role_arn -> + # Keyed by what the role is assumed with as well as the role: a + # different identity may not be allowed to, or be given less. + source = if static, do: {:static, static.access_key_id}, else: :surroundings + key = {:assume_role, role_arn, Keyword.get(opts, :external_id), source} + + cached(key, fn -> + with {:ok, base} <- base_credentials(static, opts) do + assume_role(base, role_arn, opts) + end + end) + end + end + + defp static_credentials( + %{access_key_id: access_key_id, secret_access_key: secret_access_key} = static + ) + when is_binary(access_key_id) and is_binary(secret_access_key) do + %{ + access_key_id: access_key_id, + secret_access_key: secret_access_key, + session_token: blank_to_nil(Map.get(static, :session_token)) + } + end + + defp static_credentials(_static), do: nil + + defp base_credentials(nil, opts), do: from_surroundings(opts) + defp base_credentials(static, _opts), do: {:ok, static} + + # A source that isn't configured says `:none` and the next is tried; + # one that is configured but fails is an error, not a reason to fall + # through to something else, as the SDKs also treat it. + defp from_surroundings(opts) do + [&from_environment/1, &from_web_identity/1, &from_container/1, &from_instance_metadata/1] + |> Enum.reduce_while({:error, :credentials_missing}, fn source, missing -> + case source.(opts) do + :none -> {:cont, missing} + result -> {:halt, result} end - ) + end) end - defp from_environment do + defp from_environment(_opts) do access_key_id = System.get_env("AWS_ACCESS_KEY_ID") secret_access_key = System.get_env("AWS_SECRET_ACCESS_KEY") @@ -70,7 +134,40 @@ defmodule Coflux.Launchers.AwsCredentials do end end - defp from_container do + # The token file is read on every refresh rather than once, since + # what's in it is rotated by whoever projects it there. + defp from_web_identity(opts) do + token_file = System.get_env("AWS_WEB_IDENTITY_TOKEN_FILE") + role_arn = System.get_env("AWS_ROLE_ARN") + + if present?(token_file) and present?(role_arn) do + cached({:web_identity, role_arn, token_file}, fn -> + case File.read(token_file) do + {:ok, token} -> + session_name = blank_to_nil(System.get_env("AWS_ROLE_SESSION_NAME")) + + sts_request( + [ + {"Action", "AssumeRoleWithWebIdentity"}, + {"RoleArn", role_arn}, + {"RoleSessionName", session_name || @role_session_name}, + {"WebIdentityToken", String.trim(token)} + ], + nil, + opts + ) + + {:error, reason} -> + {:error, + {:assume_role, "token_unreadable", "#{token_file}: #{:file.format_error(reason)}"}} + end + end) + else + :none + end + end + + defp from_container(_opts) do url = cond do present?(System.get_env("AWS_CONTAINER_CREDENTIALS_FULL_URI")) -> @@ -112,7 +209,7 @@ defmodule Coflux.Launchers.AwsCredentials do if token, do: [{"authorization", token}], else: [] end - defp from_instance_metadata do + defp from_instance_metadata(_opts) do if System.get_env("AWS_EC2_METADATA_DISABLED") in ["true", "1"] do :none else @@ -165,6 +262,152 @@ defmodule Coflux.Launchers.AwsCredentials do end end + # --- STS --- + + defp assume_role(base, role_arn, opts) do + params = [ + {"Action", "AssumeRole"}, + {"RoleArn", role_arn}, + {"RoleSessionName", @role_session_name} + ] + + params = + case Keyword.get(opts, :external_id) do + nil -> params + external_id -> params ++ [{"ExternalId", external_id}] + end + + sts_request(params, base, opts) + end + + # STS speaks the AWS Query protocol: form-encoded parameters in a POST, + # and XML back. The request is signed with `credentials`, or not at all + # when there are none (a web identity token is its own authentication). + defp sts_request(params, credentials, opts) do + region = sts_region(opts) + + request = + [ + method: :post, + url: sts_endpoint(region), + form: [{"Version", @sts_version} | params], + retry: false, + decode_body: false, + connect_options: [timeout: @sts_connect_timeout_ms], + receive_timeout: @sts_receive_timeout_ms + ] + |> sign(credentials, region) + |> Keyword.merge(Keyword.get(opts, :req_options, [])) + + case Req.request(request) do + {:ok, %{status: 200, body: body}} when is_binary(body) -> + parse_sts_credentials(body) + + {:ok, %{status: status, body: body}} -> + body = if is_binary(body), do: body, else: "" + code = xml_text(body, "Code") || "status_#{status}" + {:error, {:assume_role, code, xml_text(body, "Message")}} + + {:error, _exception} -> + {:error, {:assume_role, "request_failed", nil}} + end + end + + defp sign(request, nil, _region), do: request + + defp sign(request, credentials, region) do + sigv4 = [ + service: "sts", + region: region || "us-east-1", + access_key_id: credentials.access_key_id, + secret_access_key: credentials.secret_access_key + ] + + sigv4 = + case credentials[:session_token] do + nil -> sigv4 + token -> Keyword.put(sigv4, :token, token) + end + + Keyword.put(request, :aws_sigv4, sigv4) + end + + # The pool's region, as the SDKs would use the one configured, or the + # server's own; either issues tokens good everywhere. Without one the + # global endpoint does. + defp sts_region(opts) do + Keyword.get(opts, :region) || + blank_to_nil(System.get_env("AWS_REGION")) || + blank_to_nil(System.get_env("AWS_DEFAULT_REGION")) + end + + defp sts_endpoint(region) do + blank_to_nil(System.get_env("AWS_ENDPOINT_URL_STS")) || + if region, do: "https://sts.#{region}.amazonaws.com/", else: "https://sts.amazonaws.com/" + end + + # The response is a small, flat document, and the values wanted from it + # never contain markup, so it isn't worth parsing properly. + defp parse_sts_credentials(body) do + with access_key_id when is_binary(access_key_id) <- xml_text(body, "AccessKeyId"), + secret_access_key when is_binary(secret_access_key) <- xml_text(body, "SecretAccessKey"), + session_token when is_binary(session_token) <- xml_text(body, "SessionToken"), + expiration when is_binary(expiration) <- xml_text(body, "Expiration"), + {:ok, expires_at, _offset} <- DateTime.from_iso8601(expiration) do + {:ok, + %{ + access_key_id: access_key_id, + secret_access_key: secret_access_key, + session_token: session_token, + expires_at: DateTime.to_unix(expires_at) + }} + else + _ -> {:error, {:assume_role, "unexpected_response", nil}} + end + end + + defp xml_text(body, element) do + case Regex.run(~r{<#{element}>([^<]*)}, body, capture: :all_but_first) do + [text] -> xml_unescape(text) + nil -> nil + end + end + + defp xml_unescape(text) do + text + |> String.replace("<", "<") + |> String.replace(">", ">") + |> String.replace(""", "\"") + |> String.replace("'", "'") + |> String.replace("&", "&") + end + + # --- Cache --- + + # Two callers missing at once both fetch, and the second to finish + # wins; either result is good, so that costs less than coordinating. + defp cached(key, fetch) do + now = System.system_time(:second) + + case Cache.get(key) do + {:ok, %{expires_at: expires_at} = credentials} + when expires_at - now > @refresh_margin_seconds -> + {:ok, credentials} + + _ -> + case fetch.() do + {:ok, credentials} -> + Cache.put(key, credentials, now) + {:ok, credentials} + + other -> + other + end + end + end + + # --- Helpers --- + defp present?(value), do: is_binary(value) and value != "" defp blank_to_nil(value), do: if(present?(value), do: value, else: nil) diff --git a/server/lib/coflux/launchers/aws_credentials/cache.ex b/server/lib/coflux/launchers/aws_credentials/cache.ex new file mode 100644 index 00000000..9fa4441e --- /dev/null +++ b/server/lib/coflux/launchers/aws_credentials/cache.ex @@ -0,0 +1,51 @@ +defmodule Coflux.Launchers.AwsCredentials.Cache do + @moduledoc """ + Holds credentials STS has issued until they expire. + + The process only owns the table: launcher tasks read and write it + directly, so a lookup never waits on anything. Without the process (as + under test) nothing is cached, and a lookup finds nothing. + """ + + use GenServer + + @table :coflux_aws_credentials + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc "Whatever is held under `key`, expired or not; `:none` otherwise." + def get(key) do + if table?() do + case :ets.lookup(@table, key) do + [{^key, _expires_at, credentials}] -> {:ok, credentials} + [] -> :none + end + else + :none + end + end + + @doc """ + Holds `credentials` under `key` until their `expires_at`, throwing out + whatever else has expired by `now` while here, so entries for roles + and identities no longer in use don't pile up. + """ + def put(key, %{expires_at: expires_at} = credentials, now) do + if table?() do + :ets.select_delete(@table, [{{:_, :"$1", :_}, [{:<, :"$1", now}], [true]}]) + :ets.insert(@table, {key, expires_at, credentials}) + end + + :ok + end + + defp table?, do: :ets.whereis(@table) != :undefined + + @impl true + def init(_opts) do + :ets.new(@table, [:named_table, :public, :set, read_concurrency: true]) + {:ok, nil} + end +end diff --git a/server/lib/coflux/launchers/ecs.ex b/server/lib/coflux/launchers/ecs.ex index 97323419..6d35cf6a 100644 --- a/server/lib/coflux/launchers/ecs.ex +++ b/server/lib/coflux/launchers/ecs.ex @@ -17,7 +17,8 @@ defmodule Coflux.EcsLauncher do Credentials are resolved on every call rather than kept with the task: from the secret the pool names, or failing that from the server's own - surroundings. See `Coflux.Launchers.AwsCredentials`. + surroundings, and then through the role the pool names, if it does. + See `Coflux.Launchers.AwsCredentials`. """ import Coflux.Launchers.Utils, only: [truncate_bytes: 2] @@ -287,6 +288,10 @@ defmodule Coflux.EcsLauncher do # with, the detail (when there is one) is the API's own message, which # for an invalid request is the only thing that says what was wrong. defp normalize_launch_error(:credentials_missing), do: {"launch_credentials_missing", nil} + + defp normalize_launch_error({:assume_role, code, message}), + do: {"launch_assume_role_failed", if(message, do: "#{code}: #{message}", else: code)} + defp normalize_launch_error(:request_failed), do: {"launch_request_failed", nil} defp normalize_launch_error(:no_container), do: {"launch_no_container", nil} defp normalize_launch_error(:unexpected_response), do: {"launch_api_error", nil} @@ -340,6 +345,7 @@ defmodule Coflux.EcsLauncher do # For poll and stop, where the error is retried rather than shown. defp describe_error(:credentials_missing), do: "credentials_missing" + defp describe_error({:assume_role, code, _message}), do: "assume_role_failed:#{code}" defp describe_error(:request_failed), do: "request_failed" defp describe_error({:api, _status, type, _message}) when is_binary(type), @@ -354,7 +360,12 @@ defmodule Coflux.EcsLauncher do defp build_conn(config) do region = Map.fetch!(config, :region) - with {:ok, credentials} <- AwsCredentials.resolve(static_credentials(config)) do + with {:ok, credentials} <- + AwsCredentials.resolve(static_credentials(config), + region: region, + role_arn: config[:role_arn], + external_id: config[:role_external_id] + ) do {:ok, %{ region: region, diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index c61e2794..0a22bc0d 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -78,6 +78,8 @@ defmodule Coflux.Topics.Pool do |> maybe_put(:assignPublicIp, Map.get(launcher, :assign_public_ip)) |> maybe_put(:platformVersion, Map.get(launcher, :platform_version)) |> maybe_put(:credentialsSecret, Map.get(launcher, :credentials_secret)) + |> maybe_put(:roleArn, Map.get(launcher, :role_arn)) + |> maybe_put(:roleExternalId, Map.get(launcher, :role_external_id)) |> maybe_put(:endpoint, Map.get(launcher, :endpoint)) :kubernetes -> diff --git a/server/test/coflux/launchers/aws_credentials_test.exs b/server/test/coflux/launchers/aws_credentials_test.exs new file mode 100644 index 00000000..cfac56a1 --- /dev/null +++ b/server/test/coflux/launchers/aws_credentials_test.exs @@ -0,0 +1,370 @@ +defmodule Coflux.Launchers.AwsCredentialsTest do + # The environment and the credentials cache are both global. + use ExUnit.Case, async: false + + alias Coflux.Launchers.AwsCredentials + alias Coflux.Launchers.AwsCredentials.Cache + + @static %{access_key_id: "AKIASTATIC", secret_access_key: "static-secret"} + @role_arn "arn:aws:iam::123456789012:role/coflux-launcher" + + @env ~w( + AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + AWS_WEB_IDENTITY_TOKEN_FILE AWS_ROLE_ARN AWS_ROLE_SESSION_NAME + AWS_REGION AWS_DEFAULT_REGION AWS_ENDPOINT_URL_STS + ) + + # Whatever AWS configuration the machine running the tests has is + # kept out of them, and put back afterwards. + setup do + saved = Map.new(@env, &{&1, System.get_env(&1)}) + Enum.each(@env, &System.delete_env/1) + + on_exit(fn -> + Enum.each(saved, fn + {name, nil} -> System.delete_env(name) + {name, value} -> System.put_env(name, value) + end) + end) + + :ok + end + + test "static credentials are used as they are when there's no role to assume" do + assert {:ok, + %{access_key_id: "AKIASTATIC", secret_access_key: "static-secret", session_token: nil}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + req_options: [adapter: refuse()] + ) + end + + describe "assuming a role" do + test "calls STS signed with the credentials found, and uses what it issues" do + assert {:ok, credentials} = + AwsCredentials.resolve(Map.put(@static, :session_token, "static-token"), + region: "eu-west-1", + role_arn: @role_arn, + external_id: "team-42", + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ) + + assert %{ + access_key_id: "ASIAASSUMED", + secret_access_key: "assumed-secret", + session_token: "assumed-token" + } = credentials + + assert_receive {:request, request} + assert to_string(request.url) == "https://sts.eu-west-1.amazonaws.com/" + + assert URI.decode_query(request.body) == %{ + "Action" => "AssumeRole", + "Version" => "2011-06-15", + "RoleArn" => @role_arn, + "RoleSessionName" => "coflux", + "ExternalId" => "team-42" + } + + assert [authorization] = Req.Request.get_header(request, "authorization") + assert authorization =~ "AWS4-HMAC-SHA256 Credential=AKIASTATIC/" + assert authorization =~ "/eu-west-1/sts/aws4_request" + assert Req.Request.get_header(request, "x-amz-security-token") == ["static-token"] + end + + test "leaves out the external ID when there isn't one" do + assert {:ok, _} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ) + + assert_receive {:request, request} + refute Map.has_key?(URI.decode_query(request.body), "ExternalId") + end + + test "reports STS refusing with its own code and message" do + body = """ + + + Sender + AccessDenied + User "AKIASTATIC" is not authorized to perform: sts:AssumeRole + + req-1 + + """ + + assert {:error, + {:assume_role, "AccessDenied", + "User \"AKIASTATIC\" is not authorized to perform: sts:AssumeRole"}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(403, body)] + ) + end + + test "reports STS being unreachable, or answering with something else" do + unreachable = fn request -> {request, %Req.TransportError{reason: :econnrefused}} end + + assert {:error, {:assume_role, "request_failed", nil}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: unreachable] + ) + + assert {:error, {:assume_role, "unexpected_response", nil}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(200, "not sts")] + ) + + assert {:error, {:assume_role, "status_503", nil}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(503, "")] + ) + end + + test "needs credentials to assume it with" do + System.put_env("AWS_EC2_METADATA_DISABLED", "true") + on_exit(fn -> System.delete_env("AWS_EC2_METADATA_DISABLED") end) + + assert {:error, :credentials_missing} = + AwsCredentials.resolve(nil, + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: refuse()] + ) + end + end + + describe "a web identity in the environment" do + @describetag :tmp_dir + + setup %{tmp_dir: dir} do + token_file = Path.join(dir, "token") + File.write!(token_file, "eyJ.web.identity\n") + System.put_env("AWS_WEB_IDENTITY_TOKEN_FILE", token_file) + System.put_env("AWS_ROLE_ARN", @role_arn) + {:ok, token_file: token_file} + end + + test "is exchanged with STS, unsigned" do + assert {:ok, %{access_key_id: "ASIAASSUMED", session_token: "assumed-token"}} = + AwsCredentials.resolve(nil, + region: "eu-west-1", + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ) + + assert_receive {:request, request} + + assert URI.decode_query(request.body) == %{ + "Action" => "AssumeRoleWithWebIdentity", + "Version" => "2011-06-15", + "RoleArn" => @role_arn, + "RoleSessionName" => "coflux", + "WebIdentityToken" => "eyJ.web.identity" + } + + assert Req.Request.get_header(request, "authorization") == [] + end + + test "is named by AWS_ROLE_SESSION_NAME, and uses the server's own region" do + System.put_env("AWS_ROLE_SESSION_NAME", "coflux-server") + System.put_env("AWS_REGION", "us-east-2") + + assert {:ok, _} = + AwsCredentials.resolve(nil, + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ) + + assert_receive {:request, request} + assert to_string(request.url) == "https://sts.us-east-2.amazonaws.com/" + assert %{"RoleSessionName" => "coflux-server"} = URI.decode_query(request.body) + end + + test "is then what a pool's role is assumed with" do + assert {:ok, _} = + AwsCredentials.resolve(nil, + region: "eu-west-1", + role_arn: "arn:aws:iam::210987654321:role/other-account", + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ) + + assert_receive {:request, first} + assert %{"Action" => "AssumeRoleWithWebIdentity"} = URI.decode_query(first.body) + + assert_receive {:request, second} + + assert %{ + "Action" => "AssumeRole", + "RoleArn" => "arn:aws:iam::210987654321:role/other-account" + } = + URI.decode_query(second.body) + + assert [authorization] = Req.Request.get_header(second, "authorization") + assert authorization =~ "Credential=ASIAASSUMED/" + end + + test "that can't be read is an error rather than something to look past", %{ + token_file: token_file + } do + File.rm!(token_file) + + assert {:error, {:assume_role, "token_unreadable", message}} = + AwsCredentials.resolve(nil, region: "eu-west-1", req_options: [adapter: refuse()]) + + assert message =~ token_file + end + + test "gives way to static credentials on the pool" do + assert {:ok, %{access_key_id: "AKIASTATIC"}} = + AwsCredentials.resolve(@static, + region: "eu-west-1", + req_options: [adapter: refuse()] + ) + end + end + + describe "with the cache running" do + setup do + start_supervised!(Cache) + :ok + end + + test "credentials from STS are reused until shortly before they expire" do + opts = [ + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ] + + assert {:ok, first} = AwsCredentials.resolve(@static, opts) + assert {:ok, ^first} = AwsCredentials.resolve(@static, opts) + + assert_receive {:request, _} + refute_received {:request, _} + end + + test "credentials about to expire are replaced" do + opts = [ + region: "eu-west-1", + role_arn: @role_arn, + req_options: [adapter: respond(200, credentials_xml(in_seconds(60)))] + ] + + assert {:ok, _} = AwsCredentials.resolve(@static, opts) + assert {:ok, _} = AwsCredentials.resolve(@static, opts) + + assert_receive {:request, _} + assert_receive {:request, _} + end + + test "the role, its external ID and what it's assumed with each key an entry" do + adapter = respond(200, credentials_xml(in_seconds(3600))) + opts = [region: "eu-west-1", role_arn: @role_arn, req_options: [adapter: adapter]] + + assert {:ok, _} = AwsCredentials.resolve(@static, opts) + assert {:ok, _} = AwsCredentials.resolve(@static, Keyword.put(opts, :external_id, "x")) + + assert {:ok, _} = + AwsCredentials.resolve( + @static, + Keyword.put(opts, :role_arn, "arn:aws:iam::123456789012:role/another") + ) + + assert {:ok, _} = AwsCredentials.resolve(%{@static | access_key_id: "AKIAOTHER"}, opts) + + # Each of those was a first call; these aren't. + assert {:ok, _} = AwsCredentials.resolve(@static, opts) + assert {:ok, _} = AwsCredentials.resolve(@static, Keyword.put(opts, :external_id, "x")) + + for _ <- 1..4, do: assert_receive({:request, _}) + refute_received {:request, _} + end + + test "a refusal isn't kept" do + opts = [region: "eu-west-1", role_arn: @role_arn] + + assert {:error, {:assume_role, "status_500", nil}} = + AwsCredentials.resolve(@static, opts ++ [req_options: [adapter: respond(500, "")]]) + + assert {:ok, _} = + AwsCredentials.resolve( + @static, + opts ++ [req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))]] + ) + end + + @tag :tmp_dir + test "a web identity's credentials are kept too", %{tmp_dir: dir} do + token_file = Path.join(dir, "token") + File.write!(token_file, "eyJ.web.identity") + System.put_env("AWS_WEB_IDENTITY_TOKEN_FILE", token_file) + System.put_env("AWS_ROLE_ARN", @role_arn) + + opts = [ + region: "eu-west-1", + req_options: [adapter: respond(200, credentials_xml(in_seconds(3600)))] + ] + + assert {:ok, first} = AwsCredentials.resolve(nil, opts) + assert {:ok, ^first} = AwsCredentials.resolve(nil, opts) + + assert_receive {:request, _} + refute_received {:request, _} + end + end + + # --- Helpers --- + + # A stand-in for STS that answers every request the same way, and + # tells the test what it was asked. + defp respond(status, body) do + test = self() + + fn request -> + send(test, {:request, request}) + {request, Req.Response.new(status: status, body: body)} + end + end + + defp refuse do + fn _request -> flunk("no request was expected") end + end + + defp credentials_xml(expiration) do + """ + + + + ASIAASSUMED + assumed-secret + assumed-token + #{expiration} + + + AROAEXAMPLE:coflux + arn:aws:sts::123456789012:assumed-role/coflux-launcher/coflux + + + + req-1 + + + """ + end + + defp in_seconds(seconds) do + DateTime.utc_now() + |> DateTime.add(seconds, :second) + |> DateTime.truncate(:second) + |> DateTime.to_iso8601() + end +end From e4d7accd6c7f2f2d4450018af9d70a7c92d5dff5 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 13:16:32 +0100 Subject: [PATCH 13/21] Have workers/pools host all modules by default --- adapters/python/coflux/__main__.py | 7 +- adapters/python/coflux/discovery.py | 37 +++++- adapters/python/tests/test_discovery.py | 98 ++++++++++++++++ cli/cmd/coflux/manifests.go | 79 +++++++++++-- cli/cmd/coflux/manifests_test.go | 65 +++++++++++ cli/cmd/coflux/pools.go | 19 ++- cli/cmd/coflux/worker.go | 19 +-- cli/internal/worker/worker.go | 3 + docs/docs/cli_config.md | 2 +- docs/docs/cli_reference.md | 13 ++- docs/docs/getting_started/workers.md | 4 +- docs/docs/pools.md | 21 +++- docs/docs/workflows.md | 4 +- server/lib/coflux/handlers/api.ex | 9 +- server/lib/coflux/launchers/docker.ex | 6 +- server/lib/coflux/launchers/ecs.ex | 19 ++- server/lib/coflux/launchers/kubernetes.ex | 6 +- server/lib/coflux/launchers/process.ex | 5 +- .../lib/coflux/orchestration/server/fleet.ex | 25 +++- .../coflux/orchestration/server/scheduler.ex | 2 +- server/test/coflux/fleet_test.exs | 44 +++++++ tests/support/adapter.py | 16 ++- tests/support/helpers.py | 3 +- tests/test_execution.py | 26 +++++ tests/test_pools.py | 108 +++++++++++++++--- 25 files changed, 551 insertions(+), 89 deletions(-) create mode 100644 adapters/python/tests/test_discovery.py create mode 100644 cli/cmd/coflux/manifests_test.go create mode 100644 server/test/coflux/fleet_test.exs diff --git a/adapters/python/coflux/__main__.py b/adapters/python/coflux/__main__.py index b5ee0da1..0a6426e2 100644 --- a/adapters/python/coflux/__main__.py +++ b/adapters/python/coflux/__main__.py @@ -26,8 +26,11 @@ def main() -> int: ) discover_parser.add_argument( "modules", - nargs="+", - help="Python modules to scan for targets", + nargs="*", + help=( + "Python modules to scan for targets; packages are scanned recursively. " + "With none, every module in the working directory is scanned." + ), ) # execute command diff --git a/adapters/python/coflux/discovery.py b/adapters/python/coflux/discovery.py index 43f0375c..76370e53 100644 --- a/adapters/python/coflux/discovery.py +++ b/adapters/python/coflux/discovery.py @@ -5,6 +5,7 @@ import dataclasses import importlib import json +import os import pkgutil import sys import traceback @@ -21,13 +22,41 @@ ) +# Top-level names that are never where targets live, and that tend to have +# import-time side effects or dependencies the worker doesn't have. +_SKIPPED_TOP_LEVEL = frozenset({"setup", "conftest", "tests", "test"}) + + +def _working_directory_modules() -> list[str]: + """The top-level modules and packages in the working directory. + + This is what a worker hosts when it's given no modules: the directory + it was started in, which ``python -m`` puts on ``sys.path``. Private + (``_``-prefixed) names and a few conventional non-targets are left out, + and packages are only found by their ``__init__.py``, as elsewhere. + """ + cwd = os.getcwd() + if cwd not in sys.path and "" not in sys.path: + sys.path.insert(0, cwd) + + return sorted( + info.name + for info in pkgutil.iter_modules([cwd]) + if not info.name.startswith("_") and info.name not in _SKIPPED_TOP_LEVEL + ) + + def _expand_modules(module_names: list[str]) -> list[str]: """Expand package names into their constituent submodules. If a name refers to a Python package (has __path__), it is recursively walked using pkgutil.walk_packages. Plain module names are passed through unchanged. Submodules whose final component starts with '_' are skipped. + No names at all means every module in the working directory. """ + if not module_names: + module_names = _working_directory_modules() + result: list[str] = [] seen: set[str] = set() @@ -75,11 +104,13 @@ def discover_targets( """Discover all targets in the specified modules. If a module name refers to a package, all submodules are scanned - recursively (private submodules starting with '_' are skipped). + recursively (private submodules starting with '_' are skipped). An + empty list scans every module in the working directory. Args: modules: List of module or package names to scan - (e.g., ["myapp.workflows", "myapp.tasks"] or just ["myapp"]) + (e.g., ["myapp.workflows", "myapp.tasks"] or just ["myapp"]), + or [] for the working directory Returns: A ``(targets, errors)`` pair — target definitions suitable for JSON @@ -192,7 +223,7 @@ def run_discovery(modules: list[str]) -> int: file would mean restarting by hand — has it. Args: - modules: List of module names to scan. + modules: List of module names to scan, or [] for the working directory. Returns: Exit code (0 for success, 1 for error). diff --git a/adapters/python/tests/test_discovery.py b/adapters/python/tests/test_discovery.py new file mode 100644 index 00000000..6e94294a --- /dev/null +++ b/adapters/python/tests/test_discovery.py @@ -0,0 +1,98 @@ +"""What discovery scans when it's given a package, or nothing at all. + +A package name covers everything under it. No names means the working +directory: its top-level modules and packages, less the private ones and +the conventional non-targets that tend to blow up on import. +""" + +from __future__ import annotations + +import importlib +import sys +import textwrap + +import pytest + +from coflux.discovery import discover_targets + +_TASK = textwrap.dedent( + """ + import coflux as cf + + @cf.task() + def {name}(): + return 1 + """ +) + + +@pytest.fixture +def project(tmp_path, monkeypatch): + """A throwaway project as the working directory, cleaned out of + ``sys.modules`` afterwards so its names don't leak between tests.""" + monkeypatch.chdir(tmp_path) + monkeypatch.syspath_prepend(str(tmp_path)) + importlib.invalidate_caches() + before = set(sys.modules) + + def write(path, source): + file = tmp_path / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(source) + + yield write + + for name in set(sys.modules) - before: + sys.modules.pop(name, None) + + +def _names(targets): + return sorted((t["module"], t["name"]) for t in targets) + + +def test_a_package_covers_its_submodules(project): + project("dpkg/__init__.py", "") + project("dpkg/flows.py", _TASK.format(name="flow")) + project("dpkg/deep/__init__.py", "") + project("dpkg/deep/jobs.py", _TASK.format(name="job")) + project("dpkg/_private.py", _TASK.format(name="hidden")) + + targets, errors = discover_targets(["dpkg"]) + + assert errors == [] + assert _names(targets) == [("dpkg.deep.jobs", "job"), ("dpkg.flows", "flow")] + + +def test_no_modules_scans_the_working_directory(project): + project("dwd_app/__init__.py", "") + project("dwd_app/flows.py", _TASK.format(name="flow")) + project("dwd_scratch.py", _TASK.format(name="scratch")) + project("_dwd_hidden.py", _TASK.format(name="hidden")) + # Skipped by name, so importing them is never attempted + project("conftest.py", "raise RuntimeError('conftest imported')") + project("setup.py", "raise RuntimeError('setup imported')") + project("tests/__init__.py", "raise RuntimeError('tests imported')") + # Not a package without __init__.py, and not a module at all + project("dwd_data/flows.py", _TASK.format(name="orphan")) + project("notes.txt", "") + + targets, errors = discover_targets([]) + + assert errors == [] + assert _names(targets) == [("dwd_app.flows", "flow"), ("dwd_scratch", "scratch")] + + +def test_an_empty_working_directory_finds_nothing(project): + project("notes.txt", "") + + assert discover_targets([]) == ([], []) + + +def test_a_broken_module_in_the_working_directory_is_reported(project): + project("dwd_ok.py", _TASK.format(name="ok")) + project("dwd_broken.py", "import does_not_exist") + + targets, errors = discover_targets([]) + + assert _names(targets) == [("dwd_ok", "ok")] + assert [e.module for e in errors] == ["dwd_broken"] diff --git a/cli/cmd/coflux/manifests.go b/cli/cmd/coflux/manifests.go index 29615268..37ebebb5 100644 --- a/cli/cmd/coflux/manifests.go +++ b/cli/cmd/coflux/manifests.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "regexp" "sort" "strings" @@ -29,6 +30,9 @@ func init() { manifestsCmd.AddCommand(manifestsInspectCmd) manifestsInspectCmd.Flags().BoolVar(&manifestsInspectWatch, "watch", false, "Watch for changes") + + manifestsDiscoverCmd.Flags().Bool("all-modules", false, "Scan every module in the working directory, ignoring 'worker.modules' in coflux.toml") + manifestsRegisterCmd.Flags().Bool("all-modules", false, "Scan every module in the working directory, ignoring 'worker.modules' in coflux.toml") } var manifestsDiscoverCmd = &cobra.Command{ @@ -37,14 +41,16 @@ var manifestsDiscoverCmd = &cobra.Command{ Long: `Discover @task and @workflow decorated functions from the specified modules. This runs the adapter's discovery process and displays the results without -registering anything with the server. If no modules are specified, uses -'worker.modules' from coflux.toml. +registering anything with the server. A package is scanned recursively. If +no modules are specified, uses 'worker.modules' from coflux.toml, or, when +that isn't set either, every module in the working directory. Examples: coflux manifests discover myapp.workflows myapp.tasks coflux manifests discover myapp coflux manifests discover -o json myapp.workflows - coflux manifests discover # Use modules from coflux.toml`, + coflux manifests discover # Use modules from coflux.toml, or all + coflux manifests discover --all-modules # All, even if coflux.toml sets modules`, RunE: runManifestsDiscover, } @@ -54,13 +60,16 @@ var manifestsRegisterCmd = &cobra.Command{ Long: `Register workflow manifests from the specified modules with the server. This discovers @task and @workflow decorated functions and registers -their definitions with the Coflux server. If no modules are specified, -uses 'worker.modules' from coflux.toml. +their definitions with the Coflux server. A package is scanned +recursively. If no modules are specified, uses 'worker.modules' from +coflux.toml, or, when that isn't set either, every module in the working +directory. Examples: coflux manifests register myapp.workflows myapp.tasks coflux manifests register myapp - coflux manifests register # Use modules from coflux.toml`, + coflux manifests register # Use modules from coflux.toml, or all + coflux manifests register --all-modules # All, even if coflux.toml sets modules`, RunE: runManifestsRegister, } @@ -88,17 +97,66 @@ Example: RunE: runManifestsInspect, } -func resolveModules(args []string, cfg *config.Config) ([]string, error) { +// resolveModules decides what discovery is pointed at. Arguments win, then +// worker.modules from coflux.toml; with neither the result is nil, and the +// adapter scans the working directory for every module it holds. The +// --all-modules flag asks for that scan explicitly - the one thing leaving +// the arguments out can't express once the config sets a list. +func resolveModules(args []string, allModules bool, cfg *config.Config) ([]string, error) { + if allModules { + if len(args) > 0 { + return nil, fmt.Errorf("--all-modules can't be combined with module arguments") + } + return nil, nil + } modules := args if len(modules) == 0 { modules = cfg.Worker.Modules } - if len(modules) == 0 { - return nil, fmt.Errorf("no modules specified; pass them as arguments or set 'worker.modules' in coflux.toml") + for _, name := range modules { + if err := validateModuleName(name); err != nil { + return nil, err + } } return modules, nil } +var moduleNameRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$`) + +// File extensions a module name can end in only by mistake: a file given +// instead of the name it's imported by, or an unquoted glob the shell +// expanded into the directory listing. +var fileExtensions = map[string]bool{ + "py": true, "toml": true, "md": true, "txt": true, "json": true, + "yaml": true, "yml": true, "cfg": true, "ini": true, "lock": true, +} + +// validateModuleName rejects what the adapter would only fail to import, +// with a hint for the likely mistakes: a file or path, or a pattern, +// where a name already covers its submodules. +func validateModuleName(name string) error { + looksLikeFile := strings.ContainsAny(name, `/\`) || + fileExtensions[strings.ToLower(name[strings.LastIndex(name, ".")+1:])] + switch { + case strings.Contains(name, "*"): + return fmt.Errorf("'%s' is not a module name: a name covers its submodules, so patterns aren't needed (pass no modules to host everything in the working directory)", name) + case looksLikeFile: + return fmt.Errorf("'%s' is a file, not a module name: pass the name it's imported by (e.g. myapp.workflows), or no modules to host everything in the working directory", name) + case moduleNameRe.MatchString(name): + return nil + default: + return fmt.Errorf("'%s' is not a module name (e.g. myapp.workflows)", name) + } +} + +// modulesLabel names a module list in logs and messages. +func modulesLabel(modules []string) string { + if len(modules) == 0 { + return "all (working directory)" + } + return strings.Join(modules, ", ") +} + func discoverTargets(cmd *cobra.Command, modules []string) (*adapter.DiscoveryManifest, error) { cfg, err := loadConfig() if err != nil { @@ -112,7 +170,8 @@ func discoverTargets(cmd *cobra.Command, modules []string) (*adapter.DiscoveryMa return nil, fmt.Errorf("no adapter configured; use --adapter or add 'worker.adapter' to coflux.toml") } - resolved, err := resolveModules(modules, cfg) + allModules, _ := cmd.Flags().GetBool("all-modules") + resolved, err := resolveModules(modules, allModules, cfg) if err != nil { return nil, err } diff --git a/cli/cmd/coflux/manifests_test.go b/cli/cmd/coflux/manifests_test.go new file mode 100644 index 00000000..84bf8239 --- /dev/null +++ b/cli/cmd/coflux/manifests_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "strings" + "testing" + + "github.com/bitroot/coflux/cli/internal/config" +) + +func TestValidateModuleName(t *testing.T) { + for _, name := range []string{"myapp", "myapp.workflows", "_private", "A1.b2"} { + if err := validateModuleName(name); err != nil { + t.Errorf("%q: unexpected error: %v", name, err) + } + } + + cases := map[string]string{ + "myapp.*": "patterns aren't needed", + "*": "patterns aren't needed", + "myapp.py": "is a file", + "./myapp/flows.py": "is a file", + "coflux.toml": "is a file", + "README.md": "is a file", + "myapp/workflows": "is a file", + "1abc": "is not a module name", + "my-app": "is not a module name", + "myapp..workflows": "is not a module name", + "myapp.workflows.": "is not a module name", + "": "is not a module name", + } + for name, want := range cases { + err := validateModuleName(name) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%q: want error containing %q, got %v", name, want, err) + } + } +} + +func TestResolveModules(t *testing.T) { + cfg := &config.Config{} + cfg.Worker.Modules = []string{"configured"} + + if got, err := resolveModules([]string{"given"}, false, cfg); err != nil || strings.Join(got, ",") != "given" { + t.Errorf("arguments should win: got %v, %v", got, err) + } + if got, err := resolveModules(nil, false, cfg); err != nil || strings.Join(got, ",") != "configured" { + t.Errorf("config should be the fallback: got %v, %v", got, err) + } + if got, err := resolveModules(nil, false, &config.Config{}); err != nil || got != nil { + t.Errorf("nothing configured should mean all (nil): got %v, %v", got, err) + } + if got, err := resolveModules(nil, true, cfg); err != nil || got != nil { + t.Errorf("--all-modules should override config: got %v, %v", got, err) + } + if _, err := resolveModules([]string{"given"}, true, cfg); err == nil { + t.Error("--all-modules with arguments should be an error") + } + if _, err := resolveModules([]string{"bad.py"}, false, cfg); err == nil { + t.Error("an invalid argument should be an error") + } + cfg.Worker.Modules = []string{"bad.py"} + if _, err := resolveModules(nil, false, cfg); err == nil { + t.Error("an invalid configured module should be an error") + } +} diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 00300671..5cb26914 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -80,8 +80,8 @@ func runPoolsList(cmd *cobra.Command, args []string) error { if l, ok := pool["launcher"].(map[string]any); ok { launcher = getString(l, "type") } - modules := "" - if m, ok := pool["modules"].([]any); ok { + modules := "(all)" + if m, ok := pool["modules"].([]any); ok && len(m) > 0 { var mods []string for _, mod := range m { if s, ok := mod.(string); ok { @@ -134,7 +134,7 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { return outputJSON(pool) } - // Modules + // Modules (none means the pool hosts everything its workers find) if modules, ok := pool["modules"].([]any); ok && len(modules) > 0 { var mods []string for _, m := range modules { @@ -143,6 +143,8 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { } } fmt.Printf("Modules: %s\n", strings.Join(mods, ", ")) + } else { + fmt.Println("Modules: (all)") } // Provides @@ -607,14 +609,14 @@ func init() { poolsCreateCmd.Flags().String("type", "", "Launcher type (kubernetes, docker, process, ecs)") _ = poolsCreateCmd.MarkFlagRequired("type") poolsCreateCmd.Flags().StringArray("set", nil, "Set a field value (key=value)") - poolsCreateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted") + poolsCreateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted (a name covers its submodules; default: all)") poolsCreateCmd.Flags().StringSlice("provides", nil, "Features that workers provide") poolsCreateCmd.Flags().StringSlice("accepts", nil, "Tags that executions must have") // pools update flags poolsUpdateCmd.Flags().StringArray("set", nil, "Set a field value (key=value)") poolsUpdateCmd.Flags().StringArray("unset", nil, "Unset a field") - poolsUpdateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted") + poolsUpdateCmd.Flags().StringSliceP("modules", "m", nil, "Modules to be hosted (a name covers its submodules; --unset modules for all)") poolsUpdateCmd.Flags().StringSlice("provides", nil, "Features that workers provide") poolsUpdateCmd.Flags().StringSlice("accepts", nil, "Tags that executions must have") poolsUpdateCmd.Flags().Bool("no-provides", false, "Clear provides") @@ -849,13 +851,6 @@ func runPoolsCreate(cmd *cobra.Command, args []string) error { } fmt.Printf("Created pool '%s'.\n", name) - - // A pool is matched to executions by module, so one with no modules - // can never launch anything. - if modules, ok := pool["modules"].([]any); !ok || len(modules) == 0 { - fmt.Fprintf(os.Stderr, "Warning: pool '%s' has no modules, so it will not be used. Set some with: coflux pools update %s --modules ...\n", name, name) - } - return nil } diff --git a/cli/cmd/coflux/worker.go b/cli/cmd/coflux/worker.go index 5d7761f3..1a90612d 100644 --- a/cli/cmd/coflux/worker.go +++ b/cli/cmd/coflux/worker.go @@ -27,13 +27,15 @@ var workerCmd = &cobra.Command{ targets, and executes workflows and tasks as assigned. Modules can be specified as arguments or via 'worker.modules' in coflux.toml. -Packages are scanned recursively for targets. +Packages are scanned recursively for targets. With neither, every module in +the working directory is hosted; --all-modules asks for that even when +coflux.toml sets a list. Examples: - coflux worker myapp - coflux worker myapp.workflows myapp.tasks + coflux worker --dev coflux worker --dev myapp - coflux worker`, + coflux worker myapp.workflows myapp.tasks + coflux worker --all-modules`, RunE: runWorker, } @@ -41,6 +43,7 @@ var ( workerWatch bool workerRegister bool workerDev bool + workerAllModules bool workerConcurrency int workerSession string workerProvides []string @@ -53,6 +56,7 @@ func init() { workerCmd.Flags().BoolVar(&workerWatch, "watch", false, "Watch for file changes and reload") workerCmd.Flags().BoolVar(&workerRegister, "register", false, "Automatically register modules with the server") workerCmd.Flags().BoolVar(&workerDev, "dev", false, "Enable development mode (implies --watch and --register)") + workerCmd.Flags().BoolVar(&workerAllModules, "all-modules", false, "Host every module in the working directory, ignoring 'worker.modules' in coflux.toml") workerCmd.Flags().IntVar(&workerConcurrency, "concurrency", 0, "Number of concurrent executors (default: CPU count + 4)") workerCmd.Flags().StringVar(&workerSession, "session", "", "Session ID (for pool-launched workers)") workerCmd.Flags().StringSliceVar(&workerProvides, "provides", nil, "Features that this worker provides (e.g., --provides gpu:A100,gpu:H100,region:eu)") @@ -100,8 +104,8 @@ func runWorker(cmd *cobra.Command, args []string) error { } cfg.Token = token - // Resolve modules: CLI args override config - modules, err := resolveModules(args, cfg) + // Resolve modules: CLI args override config; nil means all + modules, err := resolveModules(args, workerAllModules, cfg) if err != nil { return err } @@ -175,7 +179,7 @@ func runWorker(cmd *cobra.Command, args []string) error { logger.Info("starting worker", "workspace", cfg.Workspace, "host", cfg.Host, - "modules", modules, + "modules", modulesLabel(modules), "concurrency", cfg.Worker.Concurrency, "register", shouldRegister, ) @@ -281,6 +285,7 @@ func runWorkerWithWatch( logger.Info("starting worker", "workspace", cfg.Workspace, "host", cfg.Host, + "modules", modulesLabel(modules), "concurrency", cfg.Worker.Concurrency, "register", shouldRegister, ) diff --git a/cli/internal/worker/worker.go b/cli/internal/worker/worker.go index dbb5e7f9..7c856afb 100644 --- a/cli/internal/worker/worker.go +++ b/cli/internal/worker/worker.go @@ -256,6 +256,9 @@ func (w *Worker) Run(ctx context.Context, modules []string, register bool) error w.logger.Debug("discovered targets", "count", len(manifest.Targets)) if len(manifest.Targets) == 0 { + if len(modules) == 0 { + return fmt.Errorf("%w in the working directory", ErrNoTargets) + } return fmt.Errorf("%w in modules %v", ErrNoTargets, modules) } diff --git a/docs/docs/cli_config.md b/docs/docs/cli_config.md index 3df85dc0..d3ca350e 100644 --- a/docs/docs/cli_config.md +++ b/docs/docs/cli_config.md @@ -57,7 +57,7 @@ provides = ["gpu:A100", "region:eu"] | Key | Default | Description | |-----|---------|-------------| -| `worker.modules` | `[]` | Modules to load targets from (overridden by modules passed on the command line) | +| `worker.modules` | _(all)_ | Modules to load targets from; a package is scanned recursively. Unset, every module in the working directory. Overridden by modules passed on the command line, or by `--all-modules` | | `worker.concurrency` | _(CPU count + 4, max 32)_ | Maximum concurrent executions | | `worker.adapter` | `[]` | Adapter command for executing Python code | | `worker.provides` | `[]` | Features this worker provides (for pool matching) | diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index 83160113..6dc27201 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -94,11 +94,12 @@ Fetch logs for a run or specific execution. coflux worker [modules...] ``` -Start a worker. +Start a worker. Modules can be given as arguments or as `worker.modules` in `coflux.toml`; a package is scanned recursively. With neither, every module in the working directory is hosted. | Flag | Description | |------|-------------| | `--dev` | Development mode (implies `--watch` and `--register`) | +| `--all-modules` | Host every module in the working directory, even if `coflux.toml` sets `worker.modules` | | `--watch` | Watch for file changes and reload | | `--register` | Register modules with server | | `--concurrency` | Max concurrent executions (default: CPU count + 4, max 32) | @@ -162,12 +163,12 @@ Authenticate with Coflux Studio using a device authorization flow. | Command | Description | |---------|-------------| -| `manifests discover ` | Discover targets without registering | -| `manifests register ` | Register targets with the server | +| `manifests discover [modules...]` | Discover targets without registering | +| `manifests register [modules...]` | Register targets with the server | | `manifests archive ` | Archive a module | | `manifests inspect` | List registered modules and targets (`--watch`) | -All manifest commands accept `--adapter` to specify the adapter command. +All manifest commands accept `--adapter` to specify the adapter command. `discover` and `register` resolve modules the way `coflux worker` does, and take `--all-modules`. ## `coflux pools` @@ -190,7 +191,7 @@ All manifest commands accept `--adapter` to specify the adapter command. |------|-------------| | `--type` | Launcher type: `kubernetes`, `docker`, `process`, `ecs` (required) | | `--set` | Set a field (e.g., `--set image=myapp:latest`, `--set env.KEY=VALUE`) | -| `--modules`, `-m` | Modules to host | +| `--modules`, `-m` | Modules to host, comma-separated; a name covers its submodules. Unset: all | | `--provides` | Features workers provide | | `--accepts` | Tags executions must have | @@ -200,7 +201,7 @@ All manifest commands accept `--adapter` to specify the adapter command. |------|-------------| | `--set` | Set a field | | `--unset` | Unset a field | -| `--modules`, `-m` | Modules to host | +| `--modules`, `-m` | Modules to host, comma-separated; a name covers its submodules. `--unset modules` for all | | `--provides` / `--no-provides` | Set or clear provides | | `--accepts` / `--no-accepts` | Set or clear accepts | diff --git a/docs/docs/getting_started/workers.md b/docs/docs/getting_started/workers.md index 086bf8cc..6b686f5e 100644 --- a/docs/docs/getting_started/workers.md +++ b/docs/docs/getting_started/workers.md @@ -15,9 +15,11 @@ Importantly, workers can be run locally, automatically watching for code changes Start a worker with: ```bash -coflux worker --dev hello +coflux worker --dev ``` +Given no modules, the worker hosts every module in the current directory — for now, just `hello`. Modules can also be listed explicitly (`coflux worker --dev hello`), or set in `coflux.toml`. + The worker attempts to automatically detect your Python environment. It checks for virtual environments (`.venv/`, `venv/`), package managers (`poetry`, `uv`), and falls back to `uv run --with coflux` or system Python. You can also configure the adapter explicitly with `--adapter` or in `coflux.toml` (see below). The `--dev` flag (equivalent to specifying `--watch` and `--register`) enables development mode, which watches for code changes, automatically restarts the worker, and registers workflows with the server. Without it, modules need to be registered separately (e.g., using `coflux manifests register`), and the worker would need to be restarted after making code changes. diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 9c68c11f..5d50af62 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -9,7 +9,7 @@ Pools are configured using the CLI. Use `--type` to specify the launcher type an ```bash coflux pools create mypool --type process \ --set directory=/path/to/project \ - --modules myapp.workflows myapp.tasks + --modules myapp.workflows,myapp.tasks ``` ### Launcher types @@ -193,7 +193,7 @@ These fields apply to all launcher types: | Field / Flag | Description | |--------------|-------------| -| `--modules`, `-m` | Modules to host (can be specified multiple times). Module names only — wildcards aren't supported, since this is also what launched workers are told to import | +| `--modules`, `-m` | Modules to host, comma-separated (see [Modules](#modules)). Leave unset to host everything | | `--provides` | Features that workers provide (e.g., `gpu:A100`) | | `--accepts` | Tags that executions must have to be assigned to this pool | | `idleTimeout` | Seconds the pool keeps an idle worker before stopping it (default: 5). Worth raising for launchers with slow starts, such as ECS | @@ -204,6 +204,23 @@ These fields apply to all launcher types: | `env` | Environment variables (e.g., `--set env.KEY=VALUE`) | | `envSecrets` | Environment variables set from secrets (e.g., `--set envSecrets.API_KEY=api-key`) | +### Modules + +A pool is chosen for an execution by the execution's module. The pool's +module list is what its workers are started with, so a name means what it +means to a worker: the module itself and, if it's a package, everything +under it. A pool for `myapp` hosts `myapp.workflows` and `myapp.tasks` +alike. There is no pattern syntax. + +A pool with no modules hosts everything. Its workers are started with +`--all-modules`, so they host every module in their working directory +regardless of any `worker.modules` a `coflux.toml` there sets. + +Where several pools in a workspace cover the same module, one is picked at +random for each launch. To keep work apart, route it with +[`requires` and `provides`](#provides-accepts-and-requires) rather than by +module list. + ## Secrets Anything a pool needs that mustn't be written down — an API key for workers, diff --git a/docs/docs/workflows.md b/docs/docs/workflows.md index 5f529711..eb509650 100644 --- a/docs/docs/workflows.md +++ b/docs/docs/workflows.md @@ -43,7 +43,7 @@ The decorators are designed to be unimposing — decorated functions can be call ## Modules -Targets are defined in _modules_. Typically these correspond to Python modules (i.e., `.py` files). +Targets are defined in _modules_. Typically these correspond to Python modules (i.e., `.py` files). A package is scanned recursively, so naming `myapp` covers `myapp.workflows` and `myapp.tasks`. Modules are specified when starting a worker: @@ -58,6 +58,8 @@ Or in `coflux.toml`: modules = ["myapp.workflows", "myapp.tasks"] ``` +With neither, the worker hosts every module and package in the directory it's started in (leaving out `_`-prefixed names, and `setup`, `conftest` and `tests`), so in a project that holds nothing else, `coflux worker --dev` is enough. `--all-modules` asks for that explicitly when `coflux.toml` sets a list. + Each module's targets (workflows and tasks) are declared by the worker when it connects to the server so that the server knows what targets the worker is able to handle. The workflows can also be registered with the server so they appear in Studio and can be submitted. ## Stubs diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index 115df419..cb14dc8b 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -1146,11 +1146,10 @@ defmodule Coflux.Handlers.Api do end end - # A pool's modules are module names, not patterns: the same list is - # handed to the launcher as the worker's arguments, so a wildcard would - # be passed to the worker to import - nothing expands it - as well as - # matching no execution. Accepting one would mean a pool that quietly - # never runs anything, so they are validated like any other module name. + # A pool's modules are names, not patterns: the list is handed to the + # launcher as the worker's arguments, and a name already covers its + # submodules, the way discovery imports it. An empty list is no + # restriction - the pool hosts everything its workers find. defp parse_modules(value) do value = List.wrap(value) diff --git a/server/lib/coflux/launchers/docker.ex b/server/lib/coflux/launchers/docker.ex index cb529f1c..a208516b 100644 --- a/server/lib/coflux/launchers/docker.ex +++ b/server/lib/coflux/launchers/docker.ex @@ -10,7 +10,9 @@ defmodule Coflux.DockerLauncher do # VM's - so it is a default, not a fixture. @default_network_mode "host" - def launch(env, modules, config, _opts \\ %{}) do + # `args` are the worker's arguments - its modules, or `--all-modules` - + # which the image's entrypoint (`coflux worker ...`) is expected to take. + def launch(env, args, config, _opts \\ %{}) do docker_conn = parse_docker_host(config[:docker_host]) container_env = Enum.map(env, fn {k, v} -> "#{k}=#{v}" end) @@ -22,7 +24,7 @@ defmodule Coflux.DockerLauncher do %{ "Image" => Map.fetch!(config, :image), "HostConfig" => %{"NetworkMode" => network_mode}, - "Cmd" => modules, + "Cmd" => args, "Env" => container_env } ), diff --git a/server/lib/coflux/launchers/ecs.ex b/server/lib/coflux/launchers/ecs.ex index 6d35cf6a..dd9898c0 100644 --- a/server/lib/coflux/launchers/ecs.ex +++ b/server/lib/coflux/launchers/ecs.ex @@ -3,11 +3,11 @@ defmodule Coflux.EcsLauncher do Runs workers as Amazon ECS tasks. A pool names a task definition, and each worker is one task run from it, - with the container's command overridden to the modules to host and its - environment to what the worker needs to connect. Everything else about - the task - image, CPU and memory, IAM roles, logging - belongs to the - task definition, which is where ECS users expect to configure it, so the - launcher doesn't try to own it. + with the container's command overridden to the worker's arguments (the + modules to host) and its environment to what the worker needs to + connect. Everything else about the task - image, CPU and memory, IAM + roles, logging - belongs to the task definition, which is where ECS + users expect to configure it, so the launcher doesn't try to own it. ECS has no log API of its own: container output goes wherever the task definition's log configuration sends it. What a stopped task does carry @@ -51,10 +51,10 @@ defmodule Coflux.EcsLauncher do "BlockedException" ] - def launch(env, modules, config, opts \\ %{}) do + def launch(env, args, config, opts \\ %{}) do with {:ok, conn} <- build_conn(config), {:ok, container_name} <- resolve_container_name(conn, config), - {:ok, task_arn} <- run_task(conn, config, container_name, env, modules, opts) do + {:ok, task_arn} <- run_task(conn, config, container_name, env, args, opts) do {:ok, %{ task_arn: task_arn, @@ -211,14 +211,13 @@ defmodule Coflux.EcsLauncher do end end - defp run_task(conn, config, container_name, env, modules, opts) do + defp run_task(conn, config, container_name, env, args, opts) do override = %{ "name" => container_name, + "command" => args, "environment" => Enum.map(env, fn {name, value} -> %{"name" => name, "value" => value} end) } - override = if modules == [], do: override, else: Map.put(override, "command", modules) - body = %{ "cluster" => Map.fetch!(config, :cluster), diff --git a/server/lib/coflux/launchers/kubernetes.ex b/server/lib/coflux/launchers/kubernetes.ex index b0270c1c..0e13c95b 100644 --- a/server/lib/coflux/launchers/kubernetes.ex +++ b/server/lib/coflux/launchers/kubernetes.ex @@ -13,7 +13,9 @@ defmodule Coflux.KubernetesLauncher do @log_tail_lines 20 @log_max_bytes 1024 - def launch(env, modules, config, opts \\ %{}) do + # `args` are the worker's arguments - its modules, or `--all-modules` - + # which the image's entrypoint (`coflux worker ...`) is expected to take. + def launch(env, args, config, opts \\ %{}) do namespace = Map.get(config, :namespace, "default") conn = build_conn(config) job_name = generate_job_name(opts) @@ -26,7 +28,7 @@ defmodule Coflux.KubernetesLauncher do container = %{ "name" => "worker", "image" => Map.fetch!(config, :image), - "args" => modules, + "args" => args, "env" => container_env } diff --git a/server/lib/coflux/launchers/process.ex b/server/lib/coflux/launchers/process.ex index 796de70f..b389daa4 100644 --- a/server/lib/coflux/launchers/process.ex +++ b/server/lib/coflux/launchers/process.ex @@ -29,13 +29,14 @@ defmodule Coflux.ProcessLauncher do # 128 + SIGKILL. @signalled_exit_codes [143, 137] - def launch(env, modules, config, _opts \\ %{}) do + # `args` are the worker's arguments: its modules, or `--all-modules`. + def launch(env, args, config, _opts \\ %{}) do cli_path = Coflux.Config.cli_path() directory = Map.fetch!(config, :directory) # Use `exec` so the shell is replaced by the command, ensuring # the port's OS process IS the worker (not a wrapper shell). - argv = Enum.map_join([cli_path, "worker" | modules], " ", &shell_escape/1) + argv = Enum.map_join([cli_path, "worker" | args], " ", &shell_escape/1) shell_cmd = "exec #{argv}" port_env = diff --git a/server/lib/coflux/orchestration/server/fleet.ex b/server/lib/coflux/orchestration/server/fleet.ex index 793662d3..c37a9638 100644 --- a/server/lib/coflux/orchestration/server/fleet.ex +++ b/server/lib/coflux/orchestration/server/fleet.ex @@ -331,7 +331,7 @@ defmodule Coflux.Orchestration.Server.Fleet do |> Map.get(execution.workspace_id, %{}) |> Map.filter(fn {_, pool} -> Map.get(pool, :state, :active) != :disabled && - pool.launcher && execution.module in pool.modules && + pool.launcher && pool_hosts_module?(pool.modules, execution.module) && has_requirements?(merge_tag_sets(pool.provides, Map.get(pool, :accepts, %{})), requires) && satisfies_accepts?(Map.get(pool, :accepts, %{}), requires) end) @@ -341,6 +341,29 @@ defmodule Coflux.Orchestration.Server.Fleet do end end + # A pool's modules are what its workers are started with, so they mean + # what they mean to discovery: a name covers that module and, if it's a + # package, everything under it. No modules is no restriction. + def pool_hosts_module?([], _module), do: true + + def pool_hosts_module?(modules, module) do + Enum.any?(modules, fn name -> + name == module || String.starts_with?(module, name <> ".") + end) + end + + # The arguments a launched worker is started with. A pool with no + # modules hosts everything, and the worker has to be told so rather + # than left to default: it would otherwise take `worker.modules` from + # whatever coflux.toml its working directory holds, and host less than + # the server routes to it. + def worker_args(pool) do + case pool.modules do + [] -> ["--all-modules"] + modules -> modules + end + end + def process_launcher_result(state, task_ref, result) do callback = Map.fetch!(state.launcher_tasks, task_ref) diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 006c6f5c..60cafa0c 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -811,7 +811,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do :launch, [ Fleet.build_launcher_env(state, workspace_id, token, launcher), - pool.modules, + Fleet.worker_args(pool), launcher, %{pool_name: pool_name} ], diff --git a/server/test/coflux/fleet_test.exs b/server/test/coflux/fleet_test.exs new file mode 100644 index 00000000..90a5be8d --- /dev/null +++ b/server/test/coflux/fleet_test.exs @@ -0,0 +1,44 @@ +defmodule Coflux.FleetTest do + use ExUnit.Case, async: true + + alias Coflux.Orchestration.Server.Fleet + + describe "pool_hosts_module?/2" do + test "a name covers the module itself" do + assert Fleet.pool_hosts_module?(["myapp"], "myapp") + end + + test "a name covers its submodules, as discovery imports them" do + assert Fleet.pool_hosts_module?(["myapp"], "myapp.workflows") + assert Fleet.pool_hosts_module?(["myapp"], "myapp.jobs.nightly") + end + + test "a name covers only whole components" do + refute Fleet.pool_hosts_module?(["myapp"], "myapp2") + refute Fleet.pool_hosts_module?(["myapp.work"], "myapp.workflows") + end + + test "a submodule name doesn't reach its parent" do + refute Fleet.pool_hosts_module?(["myapp.workflows"], "myapp") + end + + test "any name in the list will do" do + assert Fleet.pool_hosts_module?(["other", "myapp.tasks"], "myapp.tasks.io") + refute Fleet.pool_hosts_module?(["other", "myapp.tasks"], "myapp.workflows") + end + + test "no modules is no restriction" do + assert Fleet.pool_hosts_module?([], "anything.at.all") + end + end + + describe "worker_args/1" do + test "a pool's modules are the worker's arguments" do + assert Fleet.worker_args(%{modules: ["myapp", "other"]}) == ["myapp", "other"] + end + + test "a pool with no modules tells the worker to host everything" do + assert Fleet.worker_args(%{modules: []}) == ["--all-modules"] + end + end +end diff --git a/tests/support/adapter.py b/tests/support/adapter.py index 4f1e1d91..af1d40ca 100644 --- a/tests/support/adapter.py +++ b/tests/support/adapter.py @@ -1,10 +1,13 @@ """Test adapter shim: proxies the CLI adapter protocol over a Unix socket. -discover: prints the manifest from the file specified by --manifest. +discover: prints the manifest from the file specified by --manifest, cut + down to the modules asked for the way the real adapter would be: + a name covers its submodules, and no names means everything. execute: bidirectional proxy between stdin/stdout and a Unix socket (--socket). """ import argparse +import json import os import socket import sys @@ -12,6 +15,10 @@ import time +def _hosted(module, names): + return any(module == n or module.startswith(n + ".") for n in names) + + def discover(args): # --discover-delay stands in for a worker that is slow to start: a big # import, or an image still being pulled. The worker has a session @@ -20,7 +27,12 @@ def discover(args): if args.discover_delay: time.sleep(args.discover_delay) with open(args.manifest) as f: - print(f.read(), end="") + manifest = json.load(f) + if args.modules: + manifest["targets"] = [ + t for t in manifest["targets"] if _hosted(t["module"], args.modules) + ] + print(json.dumps(manifest), end="") def execute(args): diff --git a/tests/support/helpers.py b/tests/support/helpers.py index efb3d972..253e1437 100644 --- a/tests/support/helpers.py +++ b/tests/support/helpers.py @@ -99,7 +99,8 @@ def managed_worker( Yields an Executor. The caller is responsible for interacting with the executor and polling for results. """ - modules = modules or ["test"] + if modules is None: + modules = ["test"] worker_dir.mkdir(exist_ok=True) socket_path = str(worker_dir / "executor.sock") manifest_path = str(worker_dir / "manifest.json") diff --git a/tests/test_execution.py b/tests/test_execution.py index 5a7d7661..6957850c 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -311,6 +311,32 @@ def test_multiple_modules(worker): assert ctx.result(run_id)["value"]["data"] == 10 +def test_no_modules_hosts_everything(worker): + """A worker started without modules hosts every module it finds.""" + targets = [ + workflow("app", "main"), + task("compute", "double", parameters=["x"]), + ] + + with worker(targets, modules=[], concurrency=2) as ctx: + resp = ctx.submit("app", "main") + run_id = resp["runId"] + + ex0 = ctx.executor.next_execute() + assert ex0.target == "main" + + ref = ex0.conn.submit_task(ex0.execution_id, "compute", "double", json_args(5)) + + ex1 = ctx.executor.next_execute() + assert ex1.target == "double" + ex1.conn.complete(ex1.execution_id, value=10) + + assert ex0.conn.resolve(ex0.execution_id, ref)["value"] == 10 + + ex0.conn.complete(ex0.execution_id, value=10) + assert ctx.result(run_id)["value"]["data"] == 10 + + def test_rerun_step(worker): """Completed workflow step can be re-run, producing a new execution.""" targets = [workflow("test", "my_workflow")] diff --git a/tests/test_pools.py b/tests/test_pools.py index f5fb03cb..4bb4c4a8 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -52,6 +52,18 @@ def pool_env(server, project_id, tmp_path): executor.close() +def _adapter(pool_env): + """The test adapter command, serving pool_env's manifest and socket.""" + return [ + "python3", + ADAPTER_SCRIPT, + "--manifest", + pool_env["manifest_path"], + "--socket", + pool_env["socket_path"], + ] + + def _setup_pool( pool_env, targets, modules=None, pool_name="test-pool", provides=None, **kwargs ): @@ -61,22 +73,15 @@ def _setup_pool( starts a ``coflux worker`` process pointing at the test adapter. Extra keyword arguments are forwarded to ``cli.pools_create``. """ - modules = modules or ["test"] + if modules is None: + modules = ["test"] manifest_path = pool_env["manifest_path"] - socket_path = pool_env["socket_path"] host = pool_env["host"] with open(manifest_path, "w") as f: json.dump(manifest(targets), f) - adapter = [ - "python3", - ADAPTER_SCRIPT, - "--manifest", - manifest_path, - "--socket", - socket_path, - ] + adapter = _adapter(pool_env) cli.pools_create( pool_name, @@ -580,18 +585,85 @@ def test_disable_and_enable_round_trip(self, pool_env): class TestPoolModules: - def test_wildcard_modules_are_rejected(self, pool_env): - """A pool's modules are names, not patterns. + """A pool's modules are what its workers are started with, so they + mean what they mean to discovery: a name covers its submodules, and + no modules at all is everything.""" - The same list is handed to the launcher as the worker's arguments, - so a wildcard would be passed to the worker to import as well as - matching no execution - a pool that silently never runs anything. - """ + def test_no_modules_hosts_everything(self, pool_env): + """A pool created without modules is chosen for any module, and its + workers are started with --all-modules so they host it all.""" + host = pool_env["host"] + executor = pool_env["executor"] + targets = [ + workflow("module_a", "job_a"), + workflow("module_b", "job_b"), + ] + _setup_pool(pool_env, targets, modules=[]) + + assert cli.pools_get("test-pool", host=host)["modules"] == [] + + resp_a = cli.submit("module_a/job_a", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + + ex_a = executor.next_execute(timeout=_EXEC_TIMEOUT) + assert ex_a.target == "job_a" + ex_a.conn.complete(ex_a.execution_id, value="from_a") + + result_a = poll_result(resp_a["runId"], host, timeout=_RESULT_TIMEOUT) + assert result_a["value"]["data"] == "from_a" + + resp_b = cli.submit("module_b/job_b", host=host) + ex_b = executor.next_execute(timeout=_EXEC_TIMEOUT) + assert ex_b.target == "job_b" + ex_b.conn.complete(ex_b.execution_id, value="from_b") + + result_b = poll_result(resp_b["runId"], host, timeout=_RESULT_TIMEOUT) + assert result_b["value"]["data"] == "from_b" + + launches = cli.pools_launches("test-pool", host=host) + assert len(launches) == 1 + + def test_a_package_covers_its_submodules(self, pool_env): + """A pool for 'myapp' launches for an execution in 'myapp.workflows', + since that's what a worker started with 'myapp' imports - but not + for 'myapp2', which only shares a prefix.""" + host = pool_env["host"] + executor = pool_env["executor"] + targets = [ + workflow("myapp.workflows", "job"), + workflow("myapp2", "other"), + ] + _setup_pool(pool_env, targets, modules=["myapp"]) + # Registered so it can be submitted; nothing is configured to host it. + cli.manifests_register( + "myapp2", + adapter=",".join(_adapter(pool_env)), + host=host, + ) + + resp = cli.submit("myapp.workflows/job", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + assert ex.target == "job" + ex.conn.complete(ex.execution_id, value="done") + + result = poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + assert result["value"]["data"] == "done" + + # Nothing hosts myapp2: the worker didn't declare it, and no pool + # covers it, so the run stays unassigned and nothing new launches. + cli.submit("myapp2/other", host=host) + time.sleep(3) + assert len(cli.pools_launches("test-pool", host=host)) == 1 + + def test_patterns_are_rejected(self, pool_env): + """There is no pattern syntax: a name already covers its submodules.""" host = pool_env["host"] with pytest.raises(subprocess.CalledProcessError) as exc_info: cli.pools_create( - "wildcard-pool", + "pattern-pool", type="process", modules=["myapp.*"], process_dir=str(pool_env["worker_dir"]), @@ -599,7 +671,7 @@ def test_wildcard_modules_are_rejected(self, pool_env): ) assert "bad_request" in exc_info.value.stderr - assert "wildcard-pool" not in cli.pools_list(host=host) + assert "pattern-pool" not in cli.pools_list(host=host) class TestPoolSecrets: From 0991b5583676f845565abd86bc4c569529e40618 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 15:35:05 +0100 Subject: [PATCH 14/21] Tidy duration field naming/conventions --- adapters/python/CHANGELOG.md | 4 +- adapters/python/coflux/context.py | 8 +- adapters/python/coflux/discovery.py | 4 +- adapters/python/coflux/protocol.py | 12 +-- adapters/python/coflux/target.py | 6 +- cli/CHANGELOG.md | 7 +- cli/cmd/coflux/main.go | 33 ++++++- cli/cmd/coflux/manifests.go | 18 ++-- cli/cmd/coflux/pools.go | 95 +++++++++++++++---- cli/cmd/coflux/submit.go | 21 ++-- cli/internal/adapter/protocol.go | 10 +- cli/internal/config/config.go | 25 ++--- cli/internal/worker/worker.go | 42 ++++---- docs/docs/cli_config.md | 5 +- docs/docs/cli_reference.md | 2 +- docs/docs/pools.md | 2 +- server/CHANGELOG.md | 2 + server/lib/coflux/events.ex | 2 +- server/lib/coflux/handlers/api.ex | 47 +++++---- server/lib/coflux/handlers/worker.ex | 18 ++-- .../lib/coflux/orchestration/cache_configs.ex | 10 +- server/lib/coflux/orchestration/epoch.ex | 62 ++++++------ server/lib/coflux/orchestration/manifests.ex | 83 ++++++++-------- server/lib/coflux/orchestration/models.ex | 14 +-- server/lib/coflux/orchestration/runs.ex | 60 ++++++------ server/lib/coflux/orchestration/server.ex | 30 +++--- .../coflux/orchestration/server/commands.ex | 14 ++- .../lib/coflux/orchestration/server/fleet.ex | 22 ++--- .../coflux/orchestration/server/lifecycle.ex | 12 +-- .../coflux/orchestration/server/scheduler.ex | 30 +++--- .../coflux/orchestration/server/scheduling.ex | 8 +- .../coflux/orchestration/server/snapshots.ex | 2 +- .../lib/coflux/orchestration/server/state.ex | 2 +- server/lib/coflux/orchestration/sessions.ex | 12 +-- server/lib/coflux/orchestration/streams.ex | 2 +- server/lib/coflux/orchestration/workspaces.ex | 18 ++-- server/lib/coflux/run_view.ex | 4 +- server/lib/coflux/run_view/format.ex | 6 +- server/lib/coflux/topics/manifests.ex | 8 +- server/lib/coflux/topics/pool.ex | 2 +- server/lib/coflux/topics/workflow.ex | 10 +- server/priv/migrations/orchestration/6.sql | 29 +++++- server/test/coflux/catalog_test.exs | 2 +- server/test/coflux/checkpoints_test.exs | 2 +- .../test/coflux/concurrency_permits_test.exs | 2 +- server/test/coflux/events_test.exs | 8 +- server/test/coflux/run_view_test.exs | 2 +- server/test/coflux/streams_test.exs | 2 +- .../coflux/topics/workflow_model_test.exs | 10 +- tests/support/manifest.py | 4 +- tests/support/protocol.py | 4 +- tests/test_concurrency_limits.py | 2 +- tests/test_pools.py | 18 ++-- tests/test_topics.py | 10 +- 54 files changed, 513 insertions(+), 356 deletions(-) diff --git a/adapters/python/CHANGELOG.md b/adapters/python/CHANGELOG.md index 141182ca..50ff9b37 100644 --- a/adapters/python/CHANGELOG.md +++ b/adapters/python/CHANGELOG.md @@ -1,6 +1,8 @@ ## 0.13.0 -No changes. +Changes: + +- The manifest and submission payloads name durations with their unit: `delay` and `timeout` are `delay_ms` and `timeout_ms`, matching `max_age_ms` and `backoff_min_ms`/`backoff_max_ms`, which already did. Decorator arguments are unchanged — `delay`, `timeout` and `cache` still take seconds or a `timedelta`. ## 0.12.0 diff --git a/adapters/python/coflux/context.py b/adapters/python/coflux/context.py index d8a3ed69..5b1375c4 100644 --- a/adapters/python/coflux/context.py +++ b/adapters/python/coflux/context.py @@ -241,11 +241,11 @@ def submit_execution( cache: dict[str, Any] | None = None, defer: dict[str, Any] | None = None, memo: bool | list[int] | None = None, - delay: float | None = None, + delay_ms: int | None = None, retries: dict[str, Any] | None = None, recurrent: bool = False, requires: dict[str, list[str]] | None = None, - timeout: int = 0, + timeout_ms: int = 0, streams: dict[str, Any] | None = None, concurrency: dict[str, Any] | None = None, ) -> dict[str, Any]: @@ -267,11 +267,11 @@ def submit_execution( cache=cache, defer=defer, memo=memo, - delay=delay, + delay_ms=delay_ms, retries=retries, recurrent=recurrent, requires=requires, - timeout=timeout, + timeout_ms=timeout_ms, streams=streams, concurrency=concurrency, ) diff --git a/adapters/python/coflux/discovery.py b/adapters/python/coflux/discovery.py index 76370e53..483d6a68 100644 --- a/adapters/python/coflux/discovery.py +++ b/adapters/python/coflux/discovery.py @@ -181,7 +181,7 @@ def _build_target_definition(target: Any, module_name: str) -> dict[str, Any]: result["defer"] = serialize_defer(definition.defer, definition.parameters) if definition.delay: - result["delay"] = _to_ms(definition.delay) + result["delay_ms"] = _to_ms(definition.delay) if definition.memo: result["memo"] = definition.memo @@ -190,7 +190,7 @@ def _build_target_definition(target: Any, module_name: str) -> dict[str, Any]: result["requires"] = definition.requires if definition.timeout: - result["timeout"] = _to_ms(definition.timeout) + result["timeout_ms"] = _to_ms(definition.timeout) if definition.recurrent: result["recurrent"] = True diff --git a/adapters/python/coflux/protocol.py b/adapters/python/coflux/protocol.py index e8fab35d..8ea850cc 100644 --- a/adapters/python/coflux/protocol.py +++ b/adapters/python/coflux/protocol.py @@ -193,11 +193,11 @@ def request_submit_execution( cache: dict[str, Any] | None = None, defer: dict[str, Any] | None = None, memo: bool | list[int] | None = None, - delay: float | None = None, + delay_ms: int | None = None, retries: dict[str, Any] | None = None, recurrent: bool = False, requires: dict[str, list[str]] | None = None, - timeout: int = 0, + timeout_ms: int = 0, streams: dict[str, Any] | None = None, concurrency: dict[str, Any] | None = None, ) -> int: @@ -220,16 +220,16 @@ def request_submit_execution( params["defer"] = defer if memo is not None: params["memo"] = memo - if delay is not None: - params["delay"] = delay + if delay_ms is not None: + params["delay_ms"] = delay_ms if retries is not None: params["retries"] = retries if recurrent: params["recurrent"] = recurrent if requires: params["requires"] = requires - if timeout: - params["timeout"] = timeout + if timeout_ms: + params["timeout_ms"] = timeout_ms if streams is not None: params["streams"] = streams if concurrency is not None: diff --git a/adapters/python/coflux/target.py b/adapters/python/coflux/target.py index ce22a66f..b8d3f295 100644 --- a/adapters/python/coflux/target.py +++ b/adapters/python/coflux/target.py @@ -714,11 +714,13 @@ def submit(self, *args: P.args, **kwargs: P.kwargs) -> Execution[T]: cache=cache_dict, defer=defer_dict, memo=memo_val, - delay=_to_ms(self._definition.delay) if self._definition.delay else None, + delay_ms=_to_ms(self._definition.delay) if self._definition.delay else None, retries=retries_dict, recurrent=self._definition.recurrent, requires=self._definition.requires, - timeout=_to_ms(self._definition.timeout) if self._definition.timeout else 0, + timeout_ms=_to_ms(self._definition.timeout) + if self._definition.timeout + else 0, streams=streams_dict, concurrency=concurrency_dict, ) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index e2b4b26f..12131144 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -3,9 +3,14 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`, including the `roleArn` and `roleExternalId` fields for a role to assume before calling ECS. -- Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it. +- Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it (`--set idleTimeout=5m`, exported as `idle_timeout = "5m"`). - Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. Each secret is set for one or more workspace patterns, given as a required `--workspaces`, in the same language `tokens create --workspaces` uses. +Changes: + +- Durations are written as durations. `submit --delay` takes `30s` or `5m` rather than a bare number of seconds, and no longer accepts one; `logs.flush_interval` and `metrics.flush_interval` in `coflux.toml` are duration strings (`"500ms"`) rather than floats, and a bare number is now an error rather than being read as nanoseconds. Both need editing by hand. +- Follows the server's rename of duration fields: the manifest and submission payloads carry `delayMs`, `timeoutMs`, `maxAgeMs`, `backoffMinMs` and `backoffMaxMs`, and the worker protocol carries `max_age_ms`, `backoff_min_ms` and `backoff_max_ms`. A CLI of this version needs a server of this version, as before. + ## 0.12.0 Enhancements: diff --git a/cli/cmd/coflux/main.go b/cli/cmd/coflux/main.go index a30736f4..2fad105d 100644 --- a/cli/cmd/coflux/main.go +++ b/cli/cmd/coflux/main.go @@ -4,8 +4,10 @@ import ( "fmt" "log/slog" "os" + "reflect" "runtime" "strings" + "time" "github.com/bitroot/coflux/cli/internal/config" "github.com/bitroot/coflux/cli/internal/version" @@ -49,9 +51,9 @@ func init() { viper.SetDefault("worker.concurrency", min(runtime.NumCPU()+4, 32)) viper.SetDefault("blobs.threshold", 100) viper.SetDefault("logs.batch_size", 100) - viper.SetDefault("logs.flush_interval", 0.5) + viper.SetDefault("logs.flush_interval", "500ms") viper.SetDefault("metrics.batch_size", 100) - viper.SetDefault("metrics.flush_interval", 0.2) + viper.SetDefault("metrics.flush_interval", "200ms") viper.SetDefault("log_level", "info") // Global flags @@ -136,11 +138,38 @@ func initConfig(cmd *cobra.Command, args []string) error { return nil } +var durationType = reflect.TypeOf(time.Duration(0)) + +// stringToDurationHook decodes duration settings from duration strings +// ("500ms", "2s"). A bare number is rejected rather than truncated to +// nanoseconds: before 0.13 these were written as a float of seconds, and +// silently turning `flush_interval = 0.5` into zero would be worse than +// failing. +func stringToDurationHook(from reflect.Type, to reflect.Type, data any) (any, error) { + if to != durationType || from == durationType { + return data, nil + } + if from.Kind() != reflect.String { + return nil, fmt.Errorf("expected a duration string (e.g. \"500ms\"), got %v", data) + } + value, err := time.ParseDuration(data.(string)) + if err != nil { + return nil, fmt.Errorf("invalid duration %q (expected e.g. \"500ms\")", data) + } + return value, nil +} + // loadConfig unmarshals viper config into a Config struct func loadConfig() (*config.Config, error) { cfg := &config.Config{} if err := viper.Unmarshal(cfg, func(dc *mapstructure.DecoderConfig) { dc.ErrorUnused = true + // Replaces viper's default StringToTimeDurationHookFunc with a + // stricter one; the slice hook is viper's default, kept as-is. + dc.DecodeHook = mapstructure.ComposeDecodeHookFunc( + stringToDurationHook, + mapstructure.StringToSliceHookFunc(","), + ) }); err != nil { return nil, fmt.Errorf("failed to unmarshal config: %w", err) } diff --git a/cli/cmd/coflux/manifests.go b/cli/cmd/coflux/manifests.go index 37ebebb5..a06d96f2 100644 --- a/cli/cmd/coflux/manifests.go +++ b/cli/cmd/coflux/manifests.go @@ -411,9 +411,9 @@ func buildManifests(manifest *adapter.DiscoveryManifest) map[string]map[string]a "params": t.Cache.Params, } if t.Cache.MaxAgeMs != nil { - cacheMap["maxAge"] = *t.Cache.MaxAgeMs + cacheMap["maxAgeMs"] = *t.Cache.MaxAgeMs } else { - cacheMap["maxAge"] = nil + cacheMap["maxAgeMs"] = nil } if t.Cache.Namespace != nil { cacheMap["namespace"] = *t.Cache.Namespace @@ -453,16 +453,16 @@ func buildManifests(manifest *adapter.DiscoveryManifest) map[string]map[string]a // Delay is already in milliseconds from the adapter (0 if not set - server requires integer, not nil) delay := 0 - if t.Delay != nil { - delay = int(*t.Delay) + if t.DelayMs != nil { + delay = int(*t.DelayMs) } // Build retries (nil if not set, use 0 for delay values like Python does) var retries any if t.Retries != nil { retriesMap := map[string]any{ - "backoffMin": int64(0), - "backoffMax": int64(0), + "backoffMinMs": int64(0), + "backoffMaxMs": int64(0), } if t.Retries.Limit != nil { retriesMap["limit"] = *t.Retries.Limit @@ -470,10 +470,10 @@ func buildManifests(manifest *adapter.DiscoveryManifest) map[string]map[string]a retriesMap["limit"] = nil } if t.Retries.BackoffMinMs != nil { - retriesMap["backoffMin"] = *t.Retries.BackoffMinMs + retriesMap["backoffMinMs"] = *t.Retries.BackoffMinMs } if t.Retries.BackoffMaxMs != nil { - retriesMap["backoffMax"] = *t.Retries.BackoffMaxMs + retriesMap["backoffMaxMs"] = *t.Retries.BackoffMaxMs } retries = retriesMap } @@ -495,7 +495,7 @@ func buildManifests(manifest *adapter.DiscoveryManifest) map[string]map[string]a "waitFor": waitFor, "cache": cache, "defer": defer_, - "delay": delay, + "delayMs": delay, "retries": retries, "recurrent": t.Recurrent, "requires": requires, diff --git a/cli/cmd/coflux/pools.go b/cli/cmd/coflux/pools.go index 5cb26914..2807b08f 100644 --- a/cli/cmd/coflux/pools.go +++ b/cli/cmd/coflux/pools.go @@ -158,8 +158,8 @@ func runPoolsGet(cmd *cobra.Command, args []string) error { } // Idle timeout - if idleTimeout, ok := pool["idleTimeout"].(float64); ok { - fmt.Printf("Idle timeout: %ds\n", int(idleTimeout)) + if idleTimeout, ok := pool["idleTimeoutMs"].(float64); ok { + fmt.Printf("Idle timeout: %s\n", formatDurationMs(int64(idleTimeout))) } // Launcher @@ -635,10 +635,50 @@ func parseSetValue(s string) any { // poolTopLevelFields lists field names that are pool-level (not launcher-level). var poolTopLevelFields = map[string]bool{ - "modules": true, - "provides": true, - "accepts": true, - "idleTimeout": true, + "modules": true, + "provides": true, + "accepts": true, + "idleTimeoutMs": true, +} + +// durationFields maps the name a duration is written under - on --set, +// --unset, and in an exported config - to the API field carrying it. The +// written name has no unit suffix because the written value is a duration +// ("5m"), not a count of anything; the suffix belongs on the integer the +// API is handed. +var durationFields = map[string]string{ + "idleTimeout": "idleTimeoutMs", +} + +// parseDurationMs parses a duration string into milliseconds. A bare +// number is rejected: the pool idle timeout was briefly a count of +// seconds, and reading "300" as 300ms would be a silent thousandfold +// change. +func parseDurationMs(value string) (int64, error) { + duration, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid duration %q (expected e.g. \"30s\", \"5m\")", value) + } + if duration < 0 { + return 0, fmt.Errorf("duration %q must not be negative", value) + } + return duration.Milliseconds(), nil +} + +// formatDurationMs renders milliseconds as the shortest duration string +// that parses back to the same value. time.Duration's own String() pads +// with zero components ("5m0s"), which is noise in an exported config. +func formatDurationMs(ms int64) string { + if ms == 0 { + return "0s" + } + formatted := (time.Duration(ms) * time.Millisecond).String() + for _, suffix := range []string{"h0m0s", "m0s"} { + if strings.HasSuffix(formatted, suffix) { + return strings.TrimSuffix(formatted, suffix[1:]) + } + } + return formatted } // launcherFields lists valid launcher field names. @@ -675,6 +715,9 @@ func isValidFieldName(name string) bool { if poolTopLevelFields[name] || launcherFields[name] { return true } + if _, ok := durationFields[name]; ok { + return true + } if base, _, ok := strings.Cut(name, "."); ok { return mapSubkeyFields[base] } @@ -724,6 +767,14 @@ func collectFieldOps(cmd *cobra.Command) ([]poolFieldOp, error) { if !isValidFieldName(key) { return nil, fmt.Errorf("unknown field %q", key) } + if field, ok := durationFields[key]; ok { + ms, err := parseDurationMs(val) + if err != nil { + return nil, fmt.Errorf("--set %s: %w", key, err) + } + ops = append(ops, poolFieldOp{action: "set", key: field, value: ms}) + continue + } ops = append(ops, poolFieldOp{action: "set", key: key, value: parseSetValue(val)}) } @@ -732,6 +783,9 @@ func collectFieldOps(cmd *cobra.Command) ([]poolFieldOp, error) { if !isValidFieldName(key) { return nil, fmt.Errorf("unknown field %q", key) } + if field, ok := durationFields[key]; ok { + key = field + } ops = append(ops, poolFieldOp{action: "unset", key: key}) } @@ -1150,7 +1204,11 @@ func runPoolsImport(cmd *cobra.Command, args []string) error { if !ok { return fmt.Errorf("invalid pool configuration for '%s'", name) } - desiredPools[name] = tomlPoolToAPI(pool) + converted, err := tomlPoolToAPI(pool) + if err != nil { + return fmt.Errorf("invalid pool configuration for '%s': %w", name, err) + } + desiredPools[name] = converted } // Connect and get current state @@ -1460,8 +1518,10 @@ func apiPoolToTOML(pool map[string]any) map[string]any { result["accepts"] = accepts } } - if idleTimeout, ok := pool["idleTimeout"]; ok { - result["idle_timeout"] = tomlNumber(idleTimeout) + if idleTimeout, ok := pool["idleTimeoutMs"]; ok { + if ms, ok := tomlNumber(idleTimeout).(int64); ok { + result["idle_timeout"] = formatDurationMs(ms) + } } if launcher, ok := pool["launcher"].(map[string]any); ok { result["launcher"] = apiLauncherToTOML(launcher) @@ -1497,7 +1557,7 @@ func tomlNumber(v any) any { return v } -func tomlPoolToAPI(pool map[string]any) map[string]any { +func tomlPoolToAPI(pool map[string]any) (map[string]any, error) { result := make(map[string]any) if modules, ok := pool["modules"]; ok { @@ -1510,17 +1570,20 @@ func tomlPoolToAPI(pool map[string]any) map[string]any { result["accepts"] = toStringSliceMap(accepts) } if idleTimeout, ok := pool["idle_timeout"]; ok { - // TOML int64 → JSON number - if i, ok := idleTimeout.(int64); ok { - result["idleTimeout"] = int(i) - } else { - result["idleTimeout"] = idleTimeout + text, ok := idleTimeout.(string) + if !ok { + return nil, fmt.Errorf("idle_timeout must be a duration string (e.g. \"5m\")") } + ms, err := parseDurationMs(text) + if err != nil { + return nil, fmt.Errorf("idle_timeout: %w", err) + } + result["idleTimeoutMs"] = ms } if launcher, ok := pool["launcher"].(map[string]any); ok { result["launcher"] = tomlLauncherToAPI(launcher) } - return result + return result, nil } func tomlLauncherToAPI(launcher map[string]any) map[string]any { diff --git a/cli/cmd/coflux/submit.go b/cli/cmd/coflux/submit.go index d83b8bb8..b3e8c5c9 100644 --- a/cli/cmd/coflux/submit.go +++ b/cli/cmd/coflux/submit.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "time" "github.com/spf13/cobra" "golang.org/x/term" @@ -15,7 +16,7 @@ var submitRequires []string var submitNoRequires bool var submitMemo bool var submitNoMemo bool -var submitDelay float64 +var submitDelay time.Duration var submitRetries int var submitCmd = &cobra.Command{ @@ -55,7 +56,7 @@ func init() { submitCmd.Flags().BoolVar(&submitNoRequires, "no-requires", false, "Override the workflow's requires with an empty set") submitCmd.Flags().BoolVar(&submitMemo, "memo", false, "Override the workflow to enable memoisation") submitCmd.Flags().BoolVar(&submitNoMemo, "no-memo", false, "Override the workflow to disable memoisation") - submitCmd.Flags().Float64Var(&submitDelay, "delay", 0, "Override the workflow's delay (seconds)") + submitCmd.Flags().DurationVar(&submitDelay, "delay", 0, "Override the workflow's delay (e.g. 30s, 5m)") submitCmd.Flags().IntVar(&submitRetries, "retries", 0, "Override the workflow's retry limit (0 = no retries)") submitCmd.MarkFlagsMutuallyExclusive("requires", "no-requires") submitCmd.MarkFlagsMutuallyExclusive("memo", "no-memo") @@ -139,8 +140,8 @@ func runSubmit(cmd *cobra.Command, args []string) error { if defer_, ok := workflow["defer"]; ok { options["defer"] = defer_ } - if delay, ok := workflow["delay"].(float64); ok && delay > 0 { - options["delay"] = int64(delay) + if delay, ok := workflow["delayMs"].(float64); ok && delay > 0 { + options["delayMs"] = int64(delay) } if retries, ok := workflow["retries"]; ok { options["retries"] = retries @@ -154,8 +155,8 @@ func runSubmit(cmd *cobra.Command, args []string) error { if memo, ok := workflow["memo"]; ok { options["memo"] = memo } - if timeout, ok := workflow["timeout"].(float64); ok && timeout > 0 { - options["timeout"] = int64(timeout) + if timeout, ok := workflow["timeoutMs"].(float64); ok && timeout > 0 { + options["timeoutMs"] = int64(timeout) } if streams, ok := workflow["streams"].(map[string]any); ok && streams != nil { options["streams"] = streams @@ -176,16 +177,16 @@ func runSubmit(cmd *cobra.Command, args []string) error { options["memo"] = false } if cmd.Flags().Changed("delay") { - options["delay"] = int64(submitDelay * 1000) + options["delayMs"] = submitDelay.Milliseconds() } if cmd.Flags().Changed("retries") { if submitRetries <= 0 { options["retries"] = nil } else { options["retries"] = map[string]any{ - "limit": submitRetries, - "backoffMin": 0, - "backoffMax": 0, + "limit": submitRetries, + "backoffMinMs": 0, + "backoffMaxMs": 0, } } } diff --git a/cli/internal/adapter/protocol.go b/cli/internal/adapter/protocol.go index f7195ce3..18b6f582 100644 --- a/cli/internal/adapter/protocol.go +++ b/cli/internal/adapter/protocol.go @@ -18,12 +18,12 @@ type TargetDefinition struct { Cache *CacheConfig `json:"cache,omitempty"` Retries *RetriesConfig `json:"retries,omitempty"` Defer *DeferConfig `json:"defer,omitempty"` - Delay *float64 `json:"delay,omitempty"` + DelayMs *float64 `json:"delay_ms,omitempty"` WaitFor any `json:"wait_for,omitempty"` // true or list of param indices Memo any `json:"memo,omitempty"` // true or list of param indices Requires map[string][]string `json:"requires,omitempty"` Recurrent bool `json:"recurrent,omitempty"` - Timeout int64 `json:"timeout,omitempty"` // timeout in milliseconds + TimeoutMs int64 `json:"timeout_ms,omitempty"` Streams *StreamsConfig `json:"streams,omitempty"` Concurrency *ConcurrencyConfig `json:"concurrency,omitempty"` IsStub bool `json:"is_stub,omitempty"` @@ -239,12 +239,12 @@ type SubmitExecutionParams struct { GroupID *int `json:"group_id,omitempty"` // group to associate with Cache *CacheConfig `json:"cache,omitempty"` Defer *DeferConfig `json:"defer,omitempty"` - Memo any `json:"memo,omitempty"` // true or list of param indices - Delay *float64 `json:"delay,omitempty"` // delay in seconds + Memo any `json:"memo,omitempty"` // true or list of param indices + DelayMs *float64 `json:"delay_ms,omitempty"` Retries *RetriesConfig `json:"retries,omitempty"` Recurrent bool `json:"recurrent,omitempty"` Requires map[string][]string `json:"requires,omitempty"` - Timeout int64 `json:"timeout,omitempty"` // timeout in milliseconds + TimeoutMs int64 `json:"timeout_ms,omitempty"` Streams *StreamsConfig `json:"streams,omitempty"` Concurrency *ConcurrencyConfig `json:"concurrency,omitempty"` } diff --git a/cli/internal/config/config.go b/cli/internal/config/config.go index 46a009f7..0d6b6c53 100644 --- a/cli/internal/config/config.go +++ b/cli/internal/config/config.go @@ -1,6 +1,9 @@ package config -import "strings" +import ( + "strings" + "time" +) // Config represents the coflux.toml configuration file. // Defaults are set via viper.SetDefault() in cmd/coflux/main.go. @@ -81,20 +84,20 @@ type BlobStoreConfig struct { // LogsConfig holds log storage configuration type LogsConfig struct { - Type string `mapstructure:"type"` - Token *string `mapstructure:"token"` - URL string `mapstructure:"url"` - BatchSize int `mapstructure:"batch_size"` - FlushInterval float64 `mapstructure:"flush_interval"` + Type string `mapstructure:"type"` + Token *string `mapstructure:"token"` + URL string `mapstructure:"url"` + BatchSize int `mapstructure:"batch_size"` + FlushInterval time.Duration `mapstructure:"flush_interval"` } // MetricsConfig holds metric storage configuration type MetricsConfig struct { - Type string `mapstructure:"type"` - Token *string `mapstructure:"token"` - URL string `mapstructure:"url"` - BatchSize int `mapstructure:"batch_size"` - FlushInterval float64 `mapstructure:"flush_interval"` + Type string `mapstructure:"type"` + Token *string `mapstructure:"token"` + URL string `mapstructure:"url"` + BatchSize int `mapstructure:"batch_size"` + FlushInterval time.Duration `mapstructure:"flush_interval"` } // IsSecure determines if the connection should use TLS diff --git a/cli/internal/worker/worker.go b/cli/internal/worker/worker.go index 7c856afb..6ea63268 100644 --- a/cli/internal/worker/worker.go +++ b/cli/internal/worker/worker.go @@ -310,7 +310,7 @@ func (w *Worker) Run(ctx context.Context, modules []string, register bool) error if w.cfg.Logs.Token != nil { logToken = *w.cfg.Logs.Token } - flushInterval := time.Duration(w.cfg.Logs.FlushInterval * float64(time.Second)) + flushInterval := w.cfg.Logs.FlushInterval w.logs = logstore.NewHTTPStore(logURL, logToken, w.cfg.Project, w.cfg.Logs.BatchSize, flushInterval, w.logger) defer func() { _ = w.logs.Close() }() @@ -324,7 +324,7 @@ func (w *Worker) Run(ctx context.Context, modules []string, register bool) error if metricBatchSize <= 0 { metricBatchSize = 100 } - metricFlushInterval := time.Duration(w.cfg.Metrics.FlushInterval * float64(time.Second)) + metricFlushInterval := w.cfg.Metrics.FlushInterval if metricFlushInterval <= 0 { metricFlushInterval = 500 * time.Millisecond } @@ -971,9 +971,9 @@ func (w *Worker) SubmitExecution(ctx context.Context, params *adapter.SubmitExec "params": params.Cache.Params, } if params.Cache.MaxAgeMs != nil { - cacheMap["max_age"] = *params.Cache.MaxAgeMs + cacheMap["max_age_ms"] = *params.Cache.MaxAgeMs } else { - cacheMap["max_age"] = nil + cacheMap["max_age_ms"] = nil } if params.Cache.Namespace != nil { cacheMap["namespace"] = *params.Cache.Namespace @@ -1006,18 +1006,18 @@ func (w *Worker) SubmitExecution(ctx context.Context, params *adapter.SubmitExec retriesMap["limit"] = nil } if params.Retries.BackoffMinMs != nil { - retriesMap["backoff_min"] = *params.Retries.BackoffMinMs + retriesMap["backoff_min_ms"] = *params.Retries.BackoffMinMs } if params.Retries.BackoffMaxMs != nil { - retriesMap["backoff_max"] = *params.Retries.BackoffMaxMs + retriesMap["backoff_max_ms"] = *params.Retries.BackoffMaxMs } retries = retriesMap } // Delay is already in milliseconds from the adapter var delay int64 - if params.Delay != nil { - delay = int64(*params.Delay) + if params.DelayMs != nil { + delay = int64(*params.DelayMs) } // Determine target type (default to "task" for backward compatibility) @@ -1028,8 +1028,8 @@ func (w *Worker) SubmitExecution(ctx context.Context, params *adapter.SubmitExec // Timeout is already in milliseconds from the adapter (0 = no timeout) var timeout any - if params.Timeout > 0 { - timeout = params.Timeout + if params.TimeoutMs > 0 { + timeout = params.TimeoutMs } // Streams config (buffer + idle timeout_ms defaults for streams @@ -2389,9 +2389,9 @@ func (w *Worker) buildManifests(manifest *adapter.DiscoveryManifest) map[string] "params": t.Cache.Params, } if t.Cache.MaxAgeMs != nil { - cacheMap["max_age"] = *t.Cache.MaxAgeMs + cacheMap["max_age_ms"] = *t.Cache.MaxAgeMs } else { - cacheMap["max_age"] = nil + cacheMap["max_age_ms"] = nil } if t.Cache.Namespace != nil { cacheMap["namespace"] = *t.Cache.Namespace @@ -2416,16 +2416,16 @@ func (w *Worker) buildManifests(manifest *adapter.DiscoveryManifest) map[string] // Delay is already in milliseconds from the adapter (0 if not set - server requires integer, not nil) delay := 0 - if t.Delay != nil { - delay = int(*t.Delay) + if t.DelayMs != nil { + delay = int(*t.DelayMs) } // Build retries (nil if not set) - uses snake_case for server var retries any if t.Retries != nil { retriesMap := map[string]any{ - "backoff_min": int64(0), - "backoff_max": int64(0), + "backoff_min_ms": int64(0), + "backoff_max_ms": int64(0), } if t.Retries.Limit != nil { retriesMap["limit"] = *t.Retries.Limit @@ -2433,10 +2433,10 @@ func (w *Worker) buildManifests(manifest *adapter.DiscoveryManifest) map[string] retriesMap["limit"] = nil } if t.Retries.BackoffMinMs != nil { - retriesMap["backoff_min"] = *t.Retries.BackoffMinMs + retriesMap["backoff_min_ms"] = *t.Retries.BackoffMinMs } if t.Retries.BackoffMaxMs != nil { - retriesMap["backoff_max"] = *t.Retries.BackoffMaxMs + retriesMap["backoff_max_ms"] = *t.Retries.BackoffMaxMs } retries = retriesMap } @@ -2454,7 +2454,7 @@ func (w *Worker) buildManifests(manifest *adapter.DiscoveryManifest) map[string] } // Build timeout (0 = not set, same as delay) - timeout := int(t.Timeout) + timeout := int(t.TimeoutMs) // Build streams (nil if not set) — keys snake_case to match the // Python adapter's wire format for register_manifests. The buffer @@ -2491,10 +2491,10 @@ func (w *Worker) buildManifests(manifest *adapter.DiscoveryManifest) map[string] "waitFor": waitFor, "cache": cache, "defer": defer_, - "delay": delay, + "delayMs": delay, "retries": retries, "recurrent": t.Recurrent, - "timeout": timeout, + "timeoutMs": timeout, "requires": requires, "instruction": instruction, "memo": t.Memo, diff --git a/docs/docs/cli_config.md b/docs/docs/cli_config.md index d3ca350e..122da09a 100644 --- a/docs/docs/cli_config.md +++ b/docs/docs/cli_config.md @@ -74,9 +74,12 @@ See [Blobs](./blobs.md) for detailed blob store configuration. type = "http" url = "http://localhost:7777/logs" batch_size = 100 -flush_interval = 0.5 +flush_interval = "500ms" ``` +`flush_interval` is a duration (`"500ms"`, `"2s"`). Log and metric batches +are flushed when they fill or when this elapses, whichever comes first. + ## Environment variables All configuration keys can be set via environment variables with the `COFLUX_` prefix: diff --git a/docs/docs/cli_reference.md b/docs/docs/cli_reference.md index 6dc27201..0ce64f52 100644 --- a/docs/docs/cli_reference.md +++ b/docs/docs/cli_reference.md @@ -37,7 +37,7 @@ Arguments are passed as JSON strings. | `--requires` | Override requires tags (can be repeated) | | `--no-requires` | Clear requires | | `--memo` / `--no-memo` | Override memoisation | -| `--delay` | Override delay (seconds) | +| `--delay` | Override delay, as a duration (e.g. `30s`, `5m`) | | `--retries` | Override retry limit (0 = no retries) | ```bash diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 5d50af62..4294cd1b 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -196,7 +196,7 @@ These fields apply to all launcher types: | `--modules`, `-m` | Modules to host, comma-separated (see [Modules](#modules)). Leave unset to host everything | | `--provides` | Features that workers provide (e.g., `gpu:A100`) | | `--accepts` | Tags that executions must have to be assigned to this pool | -| `idleTimeout` | Seconds the pool keeps an idle worker before stopping it (default: 5). Worth raising for launchers with slow starts, such as ECS | +| `idleTimeout` | How long the pool keeps an idle worker before stopping it, as a duration (e.g. `30s`, `5m`; default: 5s). Worth raising for launchers with slow starts, such as ECS | | `serverHost` | Server host override for launched workers | | `serverSecure` | Use TLS for server connection | | `adapter` | Adapter command | diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 351c46e0..1d92f100 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -6,6 +6,8 @@ Enhancements: Changes: +- Durations carry their unit in their name, and are milliseconds everywhere the API and the database speak them. `delay` is `delayMs`, `timeout` is `timeoutMs`, `maxAge` is `maxAgeMs`, `backoffMin`/`backoffMax` are `backoffMinMs`/`backoffMaxMs` — on `submit_workflow`, on `register_manifests`, and in the topics that report them. The worker protocol matches, so `max_age`, `backoff_min` and `backoff_max` gain the `_ms` the adapter already sent them with. Database columns are renamed to match; no values change. +- The pool `idleTimeout` field, added earlier in this release and never published, is `idleTimeoutMs` on the API and is milliseconds rather than seconds. Its default is unchanged at 5 seconds. (The CLI still writes it as `idleTimeout`, taking a duration.) - Workspace patterns mean one thing everywhere now. A pattern selects a workspace (`development`), the workspaces under it (`development/*`, at any depth, but not `development` itself), or all of them (`*`) — the rule tokens already used, now used for a secret's workspaces too, where a bare name previously selected everything beneath it as well. Patterns that name nothing are rejected. - Setting a secret takes access containing every pattern given, whole, rather than access to any one workspace the pattern reaches — so a token for `staging` can no longer set a secret reaching `staging/feature-1`. - Secrets are set for one or more workspace patterns (`workspaces` on `set_secret` and `delete_secret`, replacing `scope`), stored once per pattern. Where patterns overlap, the nearest wins: an exact workspace, then a longer prefix, then a shorter one, then `*`. diff --git a/server/lib/coflux/events.ex b/server/lib/coflux/events.ex index d8487dcc..e9ba040d 100644 --- a/server/lib/coflux/events.ex +++ b/server/lib/coflux/events.ex @@ -169,7 +169,7 @@ defmodule Coflux.Events.StepCreated do :group_limit, :retries, :recurrent, - :timeout, + :timeout_ms, :created_at, :requires ] diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index cb14dc8b..e623bef6 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -492,10 +492,10 @@ defmodule Coflux.Handlers.Api do wait_for: {"waitFor", &parse_indexes/1}, cache: {"cache", &parse_cache/1}, defer: {"defer", &parse_defer/1}, - delay: {"delay", &parse_integer(&1, optional: true)}, + delay_ms: {"delayMs", &parse_integer(&1, optional: true)}, retries: {"retries", &parse_retries/1}, recurrent: {"recurrent", &parse_boolean(&1, optional: true)}, - timeout: {"timeout", &parse_integer(&1, optional: true)}, + timeout_ms: {"timeoutMs", &parse_integer(&1, optional: true)}, requires: {"requires", &parse_tag_set/1}, memo: {"memo", &parse_boolean(&1, optional: true)}, streams: {"streams", &parse_streams_config/1}, @@ -516,10 +516,10 @@ defmodule Coflux.Handlers.Api do wait_for: arguments[:wait_for], cache: arguments[:cache], defer: arguments[:defer], - delay: arguments[:delay] || 0, + delay_ms: arguments[:delay_ms] || 0, retries: arguments[:retries], recurrent: arguments[:recurrent] == true, - timeout: arguments[:timeout] || 0, + timeout_ms: arguments[:timeout_ms] || 0, requires: arguments[:requires], memo: arguments[:memo], streams: arguments[:streams], @@ -1637,7 +1637,7 @@ defmodule Coflux.Handlers.Api do config = if Enum.any?(provides), do: Map.put(config, "provides", provides), else: config config = if Enum.any?(accepts), do: Map.put(config, "accepts", accepts), else: config - config = maybe_put_value(config, "idleTimeout", Map.get(pool, :idle_timeout)) + config = maybe_put_value(config, "idleTimeoutMs", Map.get(pool, :idle_timeout_ms)) if pool.launcher do Map.put(config, "launcher", build_launcher_config(pool.launcher)) @@ -1729,7 +1729,7 @@ defmodule Coflux.Handlers.Api do {"modules", &parse_modules/1, :modules, []}, {"provides", &parse_tag_set/1, :provides, %{}}, {"accepts", &parse_tag_set/1, :accepts, %{}}, - {"idleTimeout", &parse_idle_timeout/1, :idle_timeout, nil}, + {"idleTimeoutMs", &parse_idle_timeout_ms/1, :idle_timeout_ms, nil}, {"launcher", &parse_launcher/1, :launcher, nil} ], {:ok, %{}}, @@ -1758,9 +1758,9 @@ defmodule Coflux.Handlers.Api do end end - # Seconds an idle worker is kept for. Zero means the next sweep. - defp parse_idle_timeout(value) when is_integer(value) and value >= 0, do: {:ok, value} - defp parse_idle_timeout(_value), do: {:error, :invalid} + # Milliseconds an idle worker is kept for. Zero means the next sweep. + defp parse_idle_timeout_ms(value) when is_integer(value) and value >= 0, do: {:ok, value} + defp parse_idle_timeout_ms(_value), do: {:error, :invalid} # Parses a partial pool update (PATCH semantics). # Only keys present in the JSON are included. A JSON null value means "unset". @@ -1771,7 +1771,7 @@ defmodule Coflux.Handlers.Api do {"modules", &parse_modules/1, :modules}, {"provides", &parse_tag_set/1, :provides}, {"accepts", &parse_tag_set/1, :accepts}, - {"idleTimeout", &parse_idle_timeout/1, :idle_timeout}, + {"idleTimeoutMs", &parse_idle_timeout_ms/1, :idle_timeout_ms}, {"launcher", &parse_launcher_patch/1, :launcher} ] @@ -2202,7 +2202,7 @@ defmodule Coflux.Handlers.Api do is_map(value) -> with {:ok, params} <- parse_indexes(Map.get(value, "params"), allow_boolean: true), - {:ok, max_age} <- parse_integer(Map.get(value, "maxAge"), optional: true), + {:ok, max_age_ms} <- parse_integer(Map.get(value, "maxAgeMs"), optional: true), # TODO: regex {:ok, namespace} <- parse_string(Map.get(value, "namespace"), optional: true, max_length: 200), @@ -2212,7 +2212,7 @@ defmodule Coflux.Handlers.Api do {:ok, %{ params: params, - max_age: max_age, + max_age_ms: max_age_ms, namespace: namespace, version: version }} @@ -2272,11 +2272,18 @@ defmodule Coflux.Handlers.Api do is_map(value) -> # limit can be nil (unlimited) or an integer - # backoff_min and backoff_max default to 0 if not provided (database requires NOT NULL) + # backoff_min_ms and backoff_max_ms default to 0 if not provided (database requires NOT NULL) with {:ok, limit} <- parse_integer(Map.get(value, "limit"), optional: true), - {:ok, backoff_min} <- parse_integer(Map.get(value, "backoffMin"), optional: true), - {:ok, backoff_max} <- parse_integer(Map.get(value, "backoffMax"), optional: true) do - {:ok, %{limit: limit, backoff_min: backoff_min || 0, backoff_max: backoff_max || 0}} + {:ok, backoff_min_ms} <- + parse_integer(Map.get(value, "backoffMinMs"), optional: true), + {:ok, backoff_max_ms} <- + parse_integer(Map.get(value, "backoffMaxMs"), optional: true) do + {:ok, + %{ + limit: limit, + backoff_min_ms: backoff_min_ms || 0, + backoff_max_ms: backoff_max_ms || 0 + }} end true -> @@ -2315,10 +2322,10 @@ defmodule Coflux.Handlers.Api do {:ok, wait_for} <- parse_indexes(Map.get(value, "waitFor")), {:ok, cache} <- parse_cache(Map.get(value, "cache")), {:ok, defer} <- parse_defer(Map.get(value, "defer")), - {:ok, delay} <- parse_integer(Map.get(value, "delay")), + {:ok, delay_ms} <- parse_integer(Map.get(value, "delayMs")), {:ok, retries} <- parse_retries(Map.get(value, "retries")), {:ok, recurrent} <- parse_boolean(Map.get(value, "recurrent"), optional: true), - {:ok, timeout} <- parse_integer(Map.get(value, "timeout"), optional: true), + {:ok, timeout_ms} <- parse_integer(Map.get(value, "timeoutMs"), optional: true), {:ok, requires} <- parse_tag_set(Map.get(value, "requires")), {:ok, memo} <- parse_boolean(Map.get(value, "memo"), optional: true), {:ok, streams} <- parse_manifest_streams(Map.get(value, "streams")), @@ -2335,10 +2342,10 @@ defmodule Coflux.Handlers.Api do wait_for: wait_for, cache: cache, defer: defer, - delay: delay, + delay_ms: delay_ms, retries: retries, recurrent: recurrent == true, - timeout: timeout || 0, + timeout_ms: timeout_ms || 0, requires: requires, memo: memo == true, streams: streams, diff --git a/server/lib/coflux/handlers/worker.ex b/server/lib/coflux/handlers/worker.ex index 78c03adc..600c5758 100644 --- a/server/lib/coflux/handlers/worker.ex +++ b/server/lib/coflux/handlers/worker.ex @@ -162,14 +162,14 @@ defmodule Coflux.Handlers.Worker do cache, defer, memo, - delay, + delay_ms, retries, recurrent, requires | rest ] = message["params"] - timeout = Enum.at(rest, 0) || 0 + timeout_ms = Enum.at(rest, 0) || 0 streams = parse_streams(Enum.at(rest, 1)) concurrency = parse_concurrency(Enum.at(rest, 2)) @@ -186,11 +186,11 @@ defmodule Coflux.Handlers.Worker do cache: parse_cache(cache), defer: parse_defer(defer), memo: memo, - delay: delay || 0, + delay_ms: delay_ms || 0, retries: parse_retries(retries), recurrent: recurrent == true, requires: requires, - timeout: timeout, + timeout_ms: timeout_ms, streams: streams, concurrency: concurrency ) do @@ -848,7 +848,7 @@ defmodule Coflux.Handlers.Worker do def websocket_info( {:execute, execution_external_id, module, target, arguments, run_id, - workspace_external_id, timeout, streams, checkpoints}, + workspace_external_id, timeout_ms, streams, checkpoints}, state ) do arguments = Enum.map(arguments, &compose_value/1) @@ -867,7 +867,7 @@ defmodule Coflux.Handlers.Worker do arguments, run_id, workspace_external_id, - timeout, + timeout_ms, compose_streams(streams), checkpoints ]) @@ -1084,7 +1084,7 @@ defmodule Coflux.Handlers.Worker do # TODO: validate %{ params: Map.fetch!(value, "params"), - max_age: Map.fetch!(value, "max_age"), + max_age_ms: Map.fetch!(value, "max_age_ms"), namespace: Map.fetch!(value, "namespace"), version: Map.fetch!(value, "version") } @@ -1113,8 +1113,8 @@ defmodule Coflux.Handlers.Worker do if value do %{ limit: Map.get(value, "limit"), - backoff_min: Map.get(value, "backoff_min"), - backoff_max: Map.get(value, "backoff_max") + backoff_min_ms: Map.get(value, "backoff_min_ms"), + backoff_max_ms: Map.get(value, "backoff_max_ms") } end end diff --git a/server/lib/coflux/orchestration/cache_configs.ex b/server/lib/coflux/orchestration/cache_configs.ex index fce6f0fa..0af367bf 100644 --- a/server/lib/coflux/orchestration/cache_configs.ex +++ b/server/lib/coflux/orchestration/cache_configs.ex @@ -14,7 +14,7 @@ defmodule Coflux.Orchestration.CacheConfigs do insert_one(db, :cache_configs, %{ hash: {:blob, hash}, params: Utils.encode_params_list(cache.params), - max_age: cache.max_age, + max_age_ms: cache.max_age_ms, namespace: cache.namespace, version: cache.version }) @@ -24,14 +24,14 @@ defmodule Coflux.Orchestration.CacheConfigs do def get_cache_config(db, cache_config_id) do case query_one( db, - "SELECT params, max_age, namespace, version FROM cache_configs WHERE id = ?1", + "SELECT params, max_age_ms, namespace, version FROM cache_configs WHERE id = ?1", {cache_config_id} ) do - {:ok, {params, max_age, namespace, version}} -> + {:ok, {params, max_age_ms, namespace, version}} -> {:ok, %{ params: Utils.decode_params_list(params), - max_age: max_age, + max_age_ms: max_age_ms, namespace: namespace, version: version }} @@ -41,7 +41,7 @@ defmodule Coflux.Orchestration.CacheConfigs do defp hash_cache_config(cache) do parts = [ Utils.encode_params_list(cache.params) || "", - if(cache.max_age, do: Integer.to_string(cache.max_age), else: ""), + if(cache.max_age_ms, do: Integer.to_string(cache.max_age_ms), else: ""), cache.namespace || "", cache.version || "" ] diff --git a/server/lib/coflux/orchestration/epoch.ex b/server/lib/coflux/orchestration/epoch.ex index 6426555b..827c8bf6 100644 --- a/server/lib/coflux/orchestration/epoch.ex +++ b/server/lib/coflux/orchestration/epoch.ex @@ -139,8 +139,8 @@ defmodule Coflux.Orchestration.Epoch do SELECT id, number, run_id, parent_id, module, target, type, priority, wait_for, cache_config_id, cache_key, defer_key, memo_key, - retry_limit, retry_backoff_min, retry_backoff_max, recurrent, delay, - timeout, requires_tag_set_id, streams_buffer, streams_timeout_ms, + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, recurrent, delay_ms, + timeout_ms, requires_tag_set_id, streams_buffer, streams_timeout_ms, concurrency_key, concurrency_limit, group_key, group_limit, created_at FROM steps WHERE run_id = ?1 @@ -153,12 +153,12 @@ defmodule Coflux.Orchestration.Epoch do Enum.reduce(steps, {%{}, %{}}, fn {old_step_id, number, _run_id, step_parent_id, module, target, type, priority, wait_for, cache_config_id, cache_key, defer_key, - memo_key, retry_limit, retry_backoff_min, - retry_backoff_max, recurrent, delay, timeout, - requires_tag_set_id, streams_buffer, - streams_timeout_ms, concurrency_key, - concurrency_limit, group_key, group_limit, - step_created_at}, + memo_key, retry_limit, retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, delay_ms, + timeout_ms, requires_tag_set_id, + streams_buffer, streams_timeout_ms, + concurrency_key, concurrency_limit, group_key, + group_limit, step_created_at}, {step_acc, exec_acc} -> # steps.parent_id is same-run internal — strict remap new_parent_id = @@ -179,11 +179,11 @@ defmodule Coflux.Orchestration.Epoch do defer_key: if(defer_key, do: {:blob, defer_key}), memo_key: if(memo_key, do: {:blob, memo_key}), retry_limit: retry_limit, - retry_backoff_min: retry_backoff_min, - retry_backoff_max: retry_backoff_max, + retry_backoff_min_ms: retry_backoff_min_ms, + retry_backoff_max_ms: retry_backoff_max_ms, recurrent: recurrent, - delay: delay, - timeout: timeout, + delay_ms: delay_ms, + timeout_ms: timeout_ms, requires_tag_set_id: ensure_tag_set(source_db, target_db, requires_tag_set_id), streams_buffer: streams_buffer, @@ -1112,7 +1112,7 @@ defmodule Coflux.Orchestration.Epoch do {:ok, rows} = query(old_db, """ SELECT s.id, s.external_id, s.workspace_id, s.worker_id, s.provides_tag_set_id, - s.accepts_tag_set_id, s.activation_timeout, s.reconnection_timeout, s.secret_hash, + s.accepts_tag_set_id, s.activation_timeout_ms, s.reconnection_timeout_ms, s.secret_hash, s.created_at, s.created_by FROM sessions AS s LEFT JOIN session_expirations AS se ON se.session_id = s.id @@ -1120,8 +1120,8 @@ defmodule Coflux.Orchestration.Epoch do """) Enum.reduce(rows, %{}, fn {old_id, ext_id, old_ws_id, old_worker_id, old_provides_tag_set_id, - old_accepts_tag_set_id, activation_timeout, reconnection_timeout, - secret_hash, created_at, created_by}, + old_accepts_tag_set_id, activation_timeout_ms, + reconnection_timeout_ms, secret_hash, created_at, created_by}, acc -> new_ws_id = Map.fetch!(workspace_ids, old_ws_id) new_worker_id = if old_worker_id, do: Map.fetch!(worker_ids, old_worker_id) @@ -1136,8 +1136,8 @@ defmodule Coflux.Orchestration.Epoch do worker_id: new_worker_id, provides_tag_set_id: new_provides_tag_set_id, accepts_tag_set_id: new_accepts_tag_set_id, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, secret_hash: if(secret_hash, do: {:blob, secret_hash}), created_at: created_at, created_by: new_created_by @@ -1830,10 +1830,10 @@ defmodule Coflux.Orchestration.Epoch do defp ensure_cache_config(_source_db, _target_db, nil), do: nil defp ensure_cache_config(source_db, target_db, old_id) do - {:ok, {hash, params, max_age, namespace, version}} = + {:ok, {hash, params, max_age_ms, namespace, version}} = query_one!( source_db, - "SELECT hash, params, max_age, namespace, version FROM cache_configs WHERE id = ?1", + "SELECT hash, params, max_age_ms, namespace, version FROM cache_configs WHERE id = ?1", {old_id} ) @@ -1846,7 +1846,7 @@ defmodule Coflux.Orchestration.Epoch do insert_one(target_db, :cache_configs, %{ hash: {:blob, hash}, params: params, - max_age: max_age, + max_age_ms: max_age_ms, namespace: namespace, version: version }) @@ -1874,17 +1874,17 @@ defmodule Coflux.Orchestration.Epoch do source_db, """ SELECT name, parameter_set_id, instruction_id, wait_for, - cache_config_id, defer_params, delay, retry_limit, - retry_backoff_min, retry_backoff_max, recurrent, requires_tag_set_id, memo + cache_config_id, defer_params, delay_ms, retry_limit, + retry_backoff_min_ms, retry_backoff_max_ms, recurrent, requires_tag_set_id, memo FROM workflows WHERE manifest_id = ?1 """, {old_id} ) - Enum.each(workflows, fn {name, ps_id, instr_id, wait_for, cc_id, defer_params, delay, - retry_limit, retry_backoff_min, retry_backoff_max, recurrent, - rts_id, memo} -> + Enum.each(workflows, fn {name, ps_id, instr_id, wait_for, cc_id, defer_params, delay_ms, + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, + recurrent, rts_id, memo} -> {:ok, _} = insert_one(target_db, :workflows, %{ manifest_id: new_id, @@ -1894,10 +1894,10 @@ defmodule Coflux.Orchestration.Epoch do wait_for: wait_for, cache_config_id: ensure_cache_config(source_db, target_db, cc_id), defer_params: defer_params, - delay: delay, + delay_ms: delay_ms, retry_limit: retry_limit, - retry_backoff_min: retry_backoff_min, - retry_backoff_max: retry_backoff_max, + retry_backoff_min_ms: retry_backoff_min_ms, + retry_backoff_max_ms: retry_backoff_max_ms, recurrent: recurrent, requires_tag_set_id: ensure_tag_set(source_db, target_db, rts_id), memo: memo @@ -1929,10 +1929,10 @@ defmodule Coflux.Orchestration.Epoch do defp ensure_pool_definition(_source_db, _target_db, nil), do: nil defp ensure_pool_definition(source_db, target_db, old_id) do - {:ok, {hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout}} = + {:ok, {hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout_ms}} = query_one!( source_db, - "SELECT hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout FROM pool_definitions WHERE id = ?1", + "SELECT hash, launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout_ms FROM pool_definitions WHERE id = ?1", {old_id} ) @@ -1951,7 +1951,7 @@ defmodule Coflux.Orchestration.Epoch do launcher_id: ensure_launcher(source_db, target_db, launcher_id), provides_tag_set_id: ensure_tag_set(source_db, target_db, provides_tag_set_id), accepts_tag_set_id: ensure_tag_set(source_db, target_db, accepts_tag_set_id), - idle_timeout: idle_timeout + idle_timeout_ms: idle_timeout_ms }) # Copy pool_definition_modules diff --git a/server/lib/coflux/orchestration/manifests.ex b/server/lib/coflux/orchestration/manifests.ex index d796022d..dd382a16 100644 --- a/server/lib/coflux/orchestration/manifests.ex +++ b/server/lib/coflux/orchestration/manifests.ex @@ -24,10 +24,10 @@ defmodule Coflux.Orchestration.Manifests do db, :workflows, {:manifest_id, :name, :instruction_id, :parameter_set_id, :wait_for, - :cache_config_id, :defer_params, :delay, :retry_limit, :retry_backoff_min, - :retry_backoff_max, :recurrent, :timeout, :requires_tag_set_id, :memo, - :streams_buffer, :streams_timeout_ms, :concurrency_limit, - :concurrency_params, :concurrency_namespace}, + :cache_config_id, :defer_params, :delay_ms, :retry_limit, + :retry_backoff_min_ms, :retry_backoff_max_ms, :recurrent, :timeout_ms, + :requires_tag_set_id, :memo, :streams_buffer, :streams_timeout_ms, + :concurrency_limit, :concurrency_params, :concurrency_namespace}, Enum.map(workflows, fn {name, workflow} -> {:ok, instruction_id} = if workflow.instruction do @@ -71,12 +71,12 @@ defmodule Coflux.Orchestration.Manifests do if(workflow.defer, do: Utils.encode_params_list(workflow.defer.params) ), - workflow.delay, + workflow.delay_ms, if(workflow.retries, do: workflow.retries.limit || -1, else: 0), - if(workflow.retries, do: workflow.retries.backoff_min, else: 0), - if(workflow.retries, do: workflow.retries.backoff_max, else: 0), + if(workflow.retries, do: workflow.retries.backoff_min_ms, else: 0), + if(workflow.retries, do: workflow.retries.backoff_max_ms, else: 0), if(workflow.recurrent, do: 1, else: 0), - workflow[:timeout] || 0, + workflow[:timeout_ms] || 0, requires_tag_set_id, if(workflow[:memo], do: 1), streams_buffer, @@ -203,7 +203,7 @@ defmodule Coflux.Orchestration.Manifests do case query_one( db, """ - SELECT w.parameter_set_id, w.instruction_id, w.wait_for, w.cache_config_id, w.defer_params, w.delay, w.retry_limit, w.retry_backoff_min, w.retry_backoff_max, w.recurrent, w.timeout, w.requires_tag_set_id, w.memo, w.streams_buffer, w.streams_timeout_ms, w.concurrency_limit, w.concurrency_params, w.concurrency_namespace + SELECT w.parameter_set_id, w.instruction_id, w.wait_for, w.cache_config_id, w.defer_params, w.delay_ms, w.retry_limit, w.retry_backoff_min_ms, w.retry_backoff_max_ms, w.recurrent, w.timeout_ms, w.requires_tag_set_id, w.memo, w.streams_buffer, w.streams_timeout_ms, w.concurrency_limit, w.concurrency_params, w.concurrency_namespace FROM workspace_manifests AS wm LEFT JOIN workflows AS w ON w.manifest_id = wm.manifest_id WHERE wm.workspace_id = ?1 AND wm.module = ?2 AND w.name = ?3 @@ -216,8 +216,8 @@ defmodule Coflux.Orchestration.Manifests do {:ok, nil} {:ok, - {parameter_set_id, instruction_id, wait_for, cache_config_id, defer_params, delay, - retry_limit, retry_backoff_min, retry_backoff_max, recurrent, timeout, + {parameter_set_id, instruction_id, wait_for, cache_config_id, defer_params, delay_ms, + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, recurrent, timeout_ms, requires_tag_set_id, memo, streams_buffer, streams_timeout_ms, concurrency_limit, concurrency_params, concurrency_namespace}} -> build_workflow( @@ -227,12 +227,12 @@ defmodule Coflux.Orchestration.Manifests do wait_for, cache_config_id, defer_params, - delay, + delay_ms, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - timeout, + timeout_ms, requires_tag_set_id, memo, streams_buffer, @@ -248,7 +248,7 @@ defmodule Coflux.Orchestration.Manifests do case query( db, """ - SELECT name, instruction_id, parameter_set_id, wait_for, cache_config_id, defer_params, delay, retry_limit, retry_backoff_min, retry_backoff_max, recurrent, timeout, requires_tag_set_id, memo, streams_buffer, streams_timeout_ms, concurrency_limit, concurrency_params, concurrency_namespace + SELECT name, instruction_id, parameter_set_id, wait_for, cache_config_id, defer_params, delay_ms, retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, recurrent, timeout_ms, requires_tag_set_id, memo, streams_buffer, streams_timeout_ms, concurrency_limit, concurrency_params, concurrency_namespace FROM workflows WHERE manifest_id = ?1 """, @@ -257,9 +257,9 @@ defmodule Coflux.Orchestration.Manifests do {:ok, rows} -> workflows = Map.new(rows, fn {name, instruction_id, parameter_set_id, wait_for, cache_config_id, - defer_params, delay, retry_limit, retry_backoff_min, - retry_backoff_max, recurrent, timeout, requires_tag_set_id, memo, - streams_buffer, streams_timeout_ms, concurrency_limit, + defer_params, delay_ms, retry_limit, retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, timeout_ms, requires_tag_set_id, + memo, streams_buffer, streams_timeout_ms, concurrency_limit, concurrency_params, concurrency_namespace} -> {:ok, workflow} = build_workflow( @@ -269,12 +269,12 @@ defmodule Coflux.Orchestration.Manifests do wait_for, cache_config_id, defer_params, - delay, + delay_ms, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - timeout, + timeout_ms, requires_tag_set_id, memo, streams_buffer, @@ -299,11 +299,14 @@ defmodule Coflux.Orchestration.Manifests do hash_parameter_set(workflow.parameters), Integer.to_string(Utils.encode_params_set(workflow.wait_for)), if(workflow.cache, do: Utils.encode_params_list(workflow.cache.params) || "", else: "-"), - if(workflow.cache[:max_age], do: Integer.to_string(workflow.cache.max_age), else: ""), + if(workflow.cache[:max_age_ms], + do: Integer.to_string(workflow.cache.max_age_ms), + else: "" + ), if(workflow.cache[:namespace], do: workflow.cache.namespace, else: ""), if(workflow.cache[:version], do: workflow.cache.version, else: ""), if(workflow.defer, do: Utils.encode_params_list(workflow.defer.params) || "", else: "-"), - Integer.to_string(workflow.delay), + Integer.to_string(workflow.delay_ms), if(workflow.retries, do: if(workflow.retries.limit, @@ -312,16 +315,16 @@ defmodule Coflux.Orchestration.Manifests do ), else: "" ), - if(workflow.retries[:backoff_min], - do: Integer.to_string(workflow.retries.backoff_min), + if(workflow.retries[:backoff_min_ms], + do: Integer.to_string(workflow.retries.backoff_min_ms), else: "" ), - if(workflow.retries[:backoff_max], - do: Integer.to_string(workflow.retries.backoff_max), + if(workflow.retries[:backoff_max_ms], + do: Integer.to_string(workflow.retries.backoff_max_ms), else: "" ), if(workflow.recurrent, do: "1", else: "0"), - Integer.to_string(workflow[:timeout] || 0), + Integer.to_string(workflow[:timeout_ms] || 0), hash_requires(workflow.requires), if(workflow[:memo], do: "1", else: "0"), workflow.instruction || "", @@ -340,12 +343,12 @@ defmodule Coflux.Orchestration.Manifests do wait_for, cache_config_id, defer_params, - delay, + delay_ms, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - timeout, + timeout_ms, requires_tag_set_id, memo, streams_buffer, @@ -387,16 +390,16 @@ defmodule Coflux.Orchestration.Manifests do retry_limit == -1 -> %{ limit: nil, - backoff_min: retry_backoff_min, - backoff_max: retry_backoff_max + backoff_min_ms: retry_backoff_min_ms, + backoff_max_ms: retry_backoff_max_ms } # positive = that many retries true -> %{ limit: retry_limit, - backoff_min: retry_backoff_min, - backoff_max: retry_backoff_max + backoff_min_ms: retry_backoff_min_ms, + backoff_max_ms: retry_backoff_max_ms } end @@ -426,10 +429,10 @@ defmodule Coflux.Orchestration.Manifests do wait_for: Utils.decode_params_set(wait_for), cache: cache, defer: defer, - delay: delay, + delay_ms: delay_ms, retries: retries, recurrent: recurrent == 1, - timeout: timeout, + timeout_ms: timeout_ms, requires: requires, memo: memo == 1, streams: streams, diff --git a/server/lib/coflux/orchestration/models.ex b/server/lib/coflux/orchestration/models.ex index d36346cd..3e6aef1f 100644 --- a/server/lib/coflux/orchestration/models.ex +++ b/server/lib/coflux/orchestration/models.ex @@ -52,11 +52,11 @@ defmodule Coflux.Orchestration.Models do :group_key, :group_limit, :retry_limit, - :retry_backoff_min, - :retry_backoff_max, + :retry_backoff_min_ms, + :retry_backoff_max_ms, :recurrent, - :delay, - :timeout, + :delay_ms, + :timeout_ms, :requires_tag_set_id, :created_at ] @@ -90,9 +90,9 @@ defmodule Coflux.Orchestration.Models do :requires_tag_set_id, :run_requires_tag_set_id, :retry_limit, - :retry_backoff_min, - :retry_backoff_max, - :timeout, + :retry_backoff_min_ms, + :retry_backoff_max_ms, + :timeout_ms, :streams_buffer, :streams_timeout_ms, :workspace_id, diff --git a/server/lib/coflux/orchestration/runs.ex b/server/lib/coflux/orchestration/runs.ex index f7bc9eb2..fec9b53b 100644 --- a/server/lib/coflux/orchestration/runs.ex +++ b/server/lib/coflux/orchestration/runs.ex @@ -37,11 +37,11 @@ defmodule Coflux.Orchestration.Runs do group_key, group_limit, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - delay, - timeout, + delay_ms, + timeout_ms, requires_tag_set_id, created_at FROM steps @@ -75,11 +75,11 @@ defmodule Coflux.Orchestration.Runs do s.group_key, s.group_limit, s.retry_limit, - s.retry_backoff_min, - s.retry_backoff_max, + s.retry_backoff_min_ms, + s.retry_backoff_max_ms, s.recurrent, - s.delay, - s.timeout, + s.delay_ms, + s.timeout_ms, s.requires_tag_set_id, s.created_at FROM steps AS s @@ -385,8 +385,8 @@ defmodule Coflux.Orchestration.Runs do memo = Keyword.get(opts, :memo) retries = Keyword.get(opts, :retries) recurrent = Keyword.get(opts, :recurrent, false) - delay = Keyword.get(opts, :delay, 0) - timeout = Keyword.get(opts, :timeout, 0) + delay_ms = Keyword.get(opts, :delay_ms, 0) + timeout_ms = Keyword.get(opts, :timeout_ms, 0) streams = Keyword.get(opts, :streams) concurrency = Keyword.get(opts, :concurrency) given_key = Keyword.get(opts, :concurrency_key) @@ -394,7 +394,7 @@ defmodule Coflux.Orchestration.Runs do requires = Keyword.get(opts, :requires) || %{} # Calculate execute_after from delay - execute_after = if delay > 0, do: now + delay + execute_after = if delay_ms > 0, do: now + delay_ms memo_key = if memo, do: build_key(memo, arguments, "#{module}:#{target}") @@ -515,11 +515,11 @@ defmodule Coflux.Orchestration.Runs do defer_key, memo_key, if(retries, do: retries.limit || -1, else: 0), - if(retries, do: retries.backoff_min || 0, else: 0), - if(retries, do: retries.backoff_max || 0, else: 0), + if(retries, do: retries.backoff_min_ms || 0, else: 0), + if(retries, do: retries.backoff_max_ms || 0, else: 0), recurrent, - delay, - timeout, + delay_ms, + timeout_ms, requires_tag_set_id, streams_buffer, streams_timeout_ms, @@ -767,9 +767,9 @@ defmodule Coflux.Orchestration.Runs do s.requires_tag_set_id, run.requires_tag_set_id AS run_requires_tag_set_id, s.retry_limit, - s.retry_backoff_min, - s.retry_backoff_max, - s.timeout, + s.retry_backoff_min_ms, + s.retry_backoff_max_ms, + s.timeout_ms, s.streams_buffer, s.streams_timeout_ms, e.workspace_id, @@ -1175,11 +1175,11 @@ defmodule Coflux.Orchestration.Runs do group_key, group_limit, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - delay, - timeout, + delay_ms, + timeout_ms, requires_tag_set_id, created_at FROM steps @@ -1584,11 +1584,11 @@ defmodule Coflux.Orchestration.Runs do defer_key, memo_key, retry_limit, - retry_backoff_min, - retry_backoff_max, + retry_backoff_min_ms, + retry_backoff_max_ms, recurrent, - delay, - timeout, + delay_ms, + timeout_ms, requires_tag_set_id, streams_buffer, streams_timeout_ms, @@ -1614,11 +1614,11 @@ defmodule Coflux.Orchestration.Runs do defer_key: if(defer_key, do: {:blob, defer_key}), memo_key: if(memo_key, do: {:blob, memo_key}), retry_limit: retry_limit, - retry_backoff_min: retry_backoff_min, - retry_backoff_max: retry_backoff_max, + retry_backoff_min_ms: retry_backoff_min_ms, + retry_backoff_max_ms: retry_backoff_max_ms, recurrent: if(recurrent, do: 1, else: 0), - delay: delay, - timeout: timeout, + delay_ms: delay_ms, + timeout_ms: timeout_ms, requires_tag_set_id: requires_tag_set_id, streams_buffer: streams_buffer, streams_timeout_ms: streams_timeout_ms, diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index b39076b1..9cf63464 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -963,18 +963,20 @@ defmodule Coflux.Orchestration.Server do defp dispatch_call({:create_session, workspace_external_id, access, opts}, state) do provides = Keyword.get(opts, :provides, %{}) accepts = Keyword.get(opts, :accepts, %{}) - activation_timeout = Keyword.get(opts, :activation_timeout, @default_activation_timeout_ms) - reconnection_timeout = - Keyword.get(opts, :reconnection_timeout, @default_reconnection_timeout_ms) + activation_timeout_ms = + Keyword.get(opts, :activation_timeout_ms, @default_activation_timeout_ms) + + reconnection_timeout_ms = + Keyword.get(opts, :reconnection_timeout_ms, @default_reconnection_timeout_ms) with {:ok, workspace_id, _} <- Permissions.require_workspace(state, workspace_external_id, access) do db_opts = [ provides: provides, accepts: accepts, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, created_by: access[:principal_id] ] @@ -998,8 +1000,8 @@ defmodule Coflux.Orchestration.Server do activated_at: nil, declared_at: nil, ready_deadline_at: nil, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, total_executions: 0 } @@ -1007,7 +1009,7 @@ defmodule Coflux.Orchestration.Server do state |> put_in([Access.key(:sessions), session_id], session) |> put_in([Access.key(:session_ids), external_session_id], session_id) - |> Fleet.schedule_session_expiry(session_id, activation_timeout) + |> Fleet.schedule_session_expiry(session_id, activation_timeout_ms) |> Listeners.maybe_schedule_idle_shutdown() {:reply, {:ok, token}, state} @@ -1070,7 +1072,7 @@ defmodule Coflux.Orchestration.Server do # fresh one; a session that has already declared keeps none. ready_deadline_at = if is_nil(session.declared_at) do - System.os_time(:millisecond) + session.activation_timeout + System.os_time(:millisecond) + session.activation_timeout_ms end state = @@ -1301,9 +1303,9 @@ defmodule Coflux.Orchestration.Server do concurrency = Keyword.get(opts, :concurrency) concurrency_limit = if concurrency, do: concurrency.limit, else: 0 retries = Keyword.get(opts, :retries) - timeout = Keyword.get(opts, :timeout, 0) - delay = Keyword.get(opts, :delay, 0) - execute_after = if delay > 0, do: created_at + delay + timeout_ms = Keyword.get(opts, :timeout_ms, 0) + delay_ms = Keyword.get(opts, :delay_ms, 0) + execute_after = if delay_ms > 0, do: created_at + delay_ms step_requires = Keyword.get(opts, :requires) || %{} run_requires = Resolve.tag_set(state.db, run.requires_tag_set_id) @@ -1358,7 +1360,7 @@ defmodule Coflux.Orchestration.Server do group_limit: group_limit, retries: retries, recurrent: recurrent, - timeout: timeout, + timeout_ms: timeout_ms, created_at: created_at, requires: step_requires }) @@ -2949,7 +2951,7 @@ defmodule Coflux.Orchestration.Server do [Access.key(:sessions), session_id], &Map.put(&1, :connection, nil) ) - |> Fleet.schedule_session_expiry(session_id, session.reconnection_timeout) + |> Fleet.schedule_session_expiry(session_id, session.reconnection_timeout_ms) |> Effects.emit(%SessionConnected{ workspace: State.workspace_external_id(state, session.workspace_id), session: session.external_id, diff --git a/server/lib/coflux/orchestration/server/commands.ex b/server/lib/coflux/orchestration/server/commands.ex index 285c0d4d..3fe33070 100644 --- a/server/lib/coflux/orchestration/server/commands.ex +++ b/server/lib/coflux/orchestration/server/commands.ex @@ -18,8 +18,18 @@ defmodule Coflux.Orchestration.Server.Commands do """ @doc "Run this execution. `streams` is the step's default stream config, or nil." - def execute(execution, module, target, arguments, run, workspace, timeout, streams, checkpoints) do - {:execute, execution, module, target, arguments, run, workspace, timeout, streams, + def execute( + execution, + module, + target, + arguments, + run, + workspace, + timeout_ms, + streams, + checkpoints + ) do + {:execute, execution, module, target, arguments, run, workspace, timeout_ms, streams, checkpoints} end diff --git a/server/lib/coflux/orchestration/server/fleet.ex b/server/lib/coflux/orchestration/server/fleet.ex index c37a9638..8ba9ff1c 100644 --- a/server/lib/coflux/orchestration/server/fleet.ex +++ b/server/lib/coflux/orchestration/server/fleet.ex @@ -7,8 +7,8 @@ defmodule Coflux.Orchestration.Server.Fleet do its connection - a worker that drops reconnects into the same session and picks up where it left off - which is why a session expires on a timer rather than on disconnect. Two timers apply: a worker that never - connects expires on its activation timeout, one that connected and went - away on its reconnection timeout. A third case is neither, and is + connects expires on its activation timeout_ms, one that connected and went + away on its reconnection timeout_ms. A third case is neither, and is handled by `Scheduler`: a worker that connects but never declares any targets is deactivated once its readiness deadline passes. @@ -198,7 +198,7 @@ defmodule Coflux.Orchestration.Server.Fleet do Until both have happened the session has never been able to accept an execution, so the fact that it isn't running one says nothing about it - being surplus - which is why the idle timeout only applies from here. + being surplus - which is why the idle timeout_ms only applies from here. Declaring an *empty* set of targets still counts: the worker answered, it just has nothing to offer, and it should be allowed to drain like any other rather than pinning its pool open forever. @@ -532,14 +532,14 @@ defmodule Coflux.Orchestration.Server.Fleet do active_sessions, state, fn {session_id, external_id, workspace_id, worker_id, provides_tag_set_id, - accepts_tag_set_id, activation_timeout, reconnection_timeout, secret_hash, created_at, - activated_at}, + accepts_tag_set_id, activation_timeout_ms, reconnection_timeout_ms, secret_hash, + created_at, activated_at}, state -> provides = Resolve.tag_set(state.db, provides_tag_set_id) accepts = Resolve.tag_set(state.db, accepts_tag_set_id) - activation_timeout = activation_timeout || @default_activation_timeout_ms - reconnection_timeout = reconnection_timeout || @default_reconnection_timeout_ms + activation_timeout_ms = activation_timeout_ms || @default_activation_timeout_ms + reconnection_timeout_ms = reconnection_timeout_ms || @default_reconnection_timeout_ms session = %{ external_id: external_id, @@ -564,8 +564,8 @@ defmodule Coflux.Orchestration.Server.Fleet do # before the restart. declared_at: nil, ready_deadline_at: nil, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, total_executions: Map.get(assignment_counts_by_session, session_id, 0) } @@ -577,9 +577,9 @@ defmodule Coflux.Orchestration.Server.Fleet do # Schedule expiry - either activation (if never connected) or reconnection (if was connected) state = if activated_at do - schedule_session_expiry(state, session_id, reconnection_timeout) + schedule_session_expiry(state, session_id, reconnection_timeout_ms) else - schedule_session_expiry(state, session_id, activation_timeout) + schedule_session_expiry(state, session_id, activation_timeout_ms) end # Link session to worker if applicable diff --git a/server/lib/coflux/orchestration/server/lifecycle.ex b/server/lib/coflux/orchestration/server/lifecycle.ex index 46358b32..58e08725 100644 --- a/server/lib/coflux/orchestration/server/lifecycle.ex +++ b/server/lib/coflux/orchestration/server/lifecycle.ex @@ -798,8 +798,8 @@ defmodule Coflux.Orchestration.Server.Lifecycle do result_retryable?(result) && step.retry_limit == -1 -> # Unlimited retries - random delay between min and max delay_ms = - step.retry_backoff_min + - :rand.uniform() * (step.retry_backoff_max - step.retry_backoff_min) + step.retry_backoff_min_ms + + :rand.uniform() * (step.retry_backoff_max_ms - step.retry_backoff_min_ms) execute_after = System.os_time(:millisecond) + delay_ms @@ -827,9 +827,9 @@ defmodule Coflux.Orchestration.Server.Lifecycle do if consecutive_failures < step.retry_limit do # TODO: add jitter (within min/max delay) delay_ms = - step.retry_backoff_min + + step.retry_backoff_min_ms + consecutive_failures / max(step.retry_limit - 1, 1) * - (step.retry_backoff_max - step.retry_backoff_min) + (step.retry_backoff_max_ms - step.retry_backoff_min_ms) execute_after = System.os_time(:millisecond) + delay_ms @@ -844,8 +844,8 @@ defmodule Coflux.Orchestration.Server.Lifecycle do step.recurrent == 1 and match?({:value, {:raw, nil, []}}, result) -> # Null return from recurrent step: schedule next iteration via :recurred execute_after = - if step.delay > 0 do - System.os_time(:millisecond) + step.delay + if step.delay_ms > 0 do + System.os_time(:millisecond) + step.delay_ms end {:ok, retry_id, _, state} = diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 60cafa0c..2bac0f8c 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -22,7 +22,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do and whose tags match what the execution requires. The pass also sweeps the fleet: sessions that have gone quiet are - polled, workers past their idle timeout are stopped, and pools short of + polled, workers past their idle timeout_ms are stopped, and pools short of their declared size get another worker launched. """ @@ -162,7 +162,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do Permissions.get_cache_workspace_ids(state, execution.workspace_id) cache = Map.fetch!(cache_configs, execution.cache_config_id) - recorded_after = if cache.max_age, do: now - cache.max_age, else: 0 + recorded_after = if cache.max_age_ms, do: now - cache.max_age_ms, else: 0 Archives.find_cached_execution_across_epochs( state, @@ -275,7 +275,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do enriched_arguments, execution.run_external_id, workspace_external_id, - execution.timeout, + execution.timeout_ms, Scheduling.build_streams_config( execution.streams_buffer, execution.streams_timeout_ms @@ -336,8 +336,8 @@ defmodule Coflux.Orchestration.Server.Scheduler do do: nil, else: execution.retry_limit ), - backoff_min: execution.retry_backoff_min, - backoff_max: execution.retry_backoff_max + backoff_min_ms: execution.retry_backoff_min_ms, + backoff_max_ms: execution.retry_backoff_max_ms } ), requires: requires, @@ -468,19 +468,19 @@ defmodule Coflux.Orchestration.Server.Scheduler do ) # Create a session for the pool-launched worker - activation_timeout = - Map.get(pool, :activation_timeout, @default_activation_timeout_ms) + activation_timeout_ms = + Map.get(pool, :activation_timeout_ms, @default_activation_timeout_ms) - reconnection_timeout = - Map.get(pool, :reconnection_timeout, @default_reconnection_timeout_ms) + reconnection_timeout_ms = + Map.get(pool, :reconnection_timeout_ms, @default_reconnection_timeout_ms) pool_accepts = Map.get(pool, :accepts, %{}) session_opts = [ provides: pool.provides, accepts: pool_accepts, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms ] {:ok, session_id, external_id, token, secret_hash, session_now} = @@ -504,15 +504,15 @@ defmodule Coflux.Orchestration.Server.Scheduler do activated_at: nil, declared_at: nil, ready_deadline_at: nil, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, total_executions: 0 } state |> put_in([Access.key(:sessions), session_id], session) |> put_in([Access.key(:session_ids), external_id], session_id) - |> Fleet.schedule_session_expiry(session_id, activation_timeout) + |> Fleet.schedule_session_expiry(session_id, activation_timeout_ms) |> Listeners.maybe_schedule_idle_shutdown() |> put_in([Access.key(:workers), worker_id], %{ external_id: worker_external_id, @@ -784,7 +784,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do # worker that takes a while to start is worth keeping warm between # runs. One whose pool doesn't say gets the default. defp worker_idle_timeout_ms(state, worker) do - case get_in(state.pools, [worker.workspace_id, worker.pool_name, :idle_timeout]) do + case get_in(state.pools, [worker.workspace_id, worker.pool_name, :idle_timeout_ms]) do seconds when is_integer(seconds) and seconds >= 0 -> seconds * 1000 _ -> @default_worker_idle_timeout_ms end diff --git a/server/lib/coflux/orchestration/server/scheduling.ex b/server/lib/coflux/orchestration/server/scheduling.ex index 2148a201..868704d7 100644 --- a/server/lib/coflux/orchestration/server/scheduling.ex +++ b/server/lib/coflux/orchestration/server/scheduling.ex @@ -110,8 +110,8 @@ defmodule Coflux.Orchestration.Server.Scheduling do attempt: attempt, created_at: created_at }} -> - delay = Keyword.get(opts, :delay, 0) - execute_after = if delay > 0, do: created_at + delay + delay_ms = Keyword.get(opts, :delay_ms, 0) + execute_after = if delay_ms > 0, do: created_at + delay_ms principal = case Principals.get_principal(state.db, created_by) do @@ -442,8 +442,8 @@ defmodule Coflux.Orchestration.Server.Scheduling do def step_retries(step) do %{ limit: if(step.retry_limit == -1, do: nil, else: step.retry_limit), - backoff_min: step.retry_backoff_min, - backoff_max: step.retry_backoff_max + backoff_min_ms: step.retry_backoff_min_ms, + backoff_max_ms: step.retry_backoff_max_ms } end diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index bcac0a40..c216ce68 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -900,7 +900,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do group_limit: step.group_limit, retries: Scheduling.step_retries(step), recurrent: step.recurrent == 1 or step.recurrent == true, - timeout: step.timeout, + timeout_ms: step.timeout_ms, created_at: step.created_at, requires: Resolve.tag_set(db, step.requires_tag_set_id) } diff --git a/server/lib/coflux/orchestration/server/state.ex b/server/lib/coflux/orchestration/server/state.ex index 1c9f2f08..1ab2e163 100644 --- a/server/lib/coflux/orchestration/server/state.ex +++ b/server/lib/coflux/orchestration/server/state.ex @@ -111,7 +111,7 @@ defmodule Coflux.Orchestration.Server.State do # why a session is separate from it and outlives it. connections: %{}, - # session_id -> %{external_id, connection, targets, queue, starting, executing, concurrency, workspace_id, provides, accepts, worker_id, last_idle_at, activated_at, declared_at, activation_timeout, reconnection_timeout} + # session_id -> %{external_id, connection, targets, queue, starting, executing, concurrency, workspace_id, provides, accepts, worker_id, last_idle_at, activated_at, declared_at, activation_timeout_ms, reconnection_timeout_ms} # # One entry per worker session. A session outlives its # connection - a worker that drops reconnects into the same one diff --git a/server/lib/coflux/orchestration/sessions.ex b/server/lib/coflux/orchestration/sessions.ex index a595b94c..f1b403da 100644 --- a/server/lib/coflux/orchestration/sessions.ex +++ b/server/lib/coflux/orchestration/sessions.ex @@ -6,8 +6,8 @@ defmodule Coflux.Orchestration.Sessions do def create_session(db, workspace_id, worker_id, opts \\ []) do provides = Keyword.get(opts, :provides) accepts = Keyword.get(opts, :accepts) - activation_timeout = Keyword.get(opts, :activation_timeout) - reconnection_timeout = Keyword.get(opts, :reconnection_timeout) + activation_timeout_ms = Keyword.get(opts, :activation_timeout_ms) + reconnection_timeout_ms = Keyword.get(opts, :reconnection_timeout_ms) created_by = Keyword.get(opts, :created_by) with_transaction(db, fn -> @@ -41,8 +41,8 @@ defmodule Coflux.Orchestration.Sessions do worker_id: worker_id, provides_tag_set_id: provides_tag_set_id, accepts_tag_set_id: accepts_tag_set_id, - activation_timeout: activation_timeout, - reconnection_timeout: reconnection_timeout, + activation_timeout_ms: activation_timeout_ms, + reconnection_timeout_ms: reconnection_timeout_ms, secret_hash: {:blob, secret_hash}, created_at: now, created_by: created_by @@ -110,8 +110,8 @@ defmodule Coflux.Orchestration.Sessions do s.worker_id, s.provides_tag_set_id, s.accepts_tag_set_id, - s.activation_timeout, - s.reconnection_timeout, + s.activation_timeout_ms, + s.reconnection_timeout_ms, s.secret_hash, s.created_at, sa.created_at AS activated_at diff --git a/server/lib/coflux/orchestration/streams.ex b/server/lib/coflux/orchestration/streams.ex index 94a72f77..e393d4f3 100644 --- a/server/lib/coflux/orchestration/streams.ex +++ b/server/lib/coflux/orchestration/streams.ex @@ -26,7 +26,7 @@ defmodule Coflux.Orchestration.Streams do recorded. The closure records the execution that closed the stream. * Only an execution registered on a stream may append to it. * The latest registrant is the stream's producer, and its registration - carries the config (buffer, timeout) in force. + carries the config (buffer, timeout_ms) in force. * Consumer cursors are kept in-memory only; re-run consumers subscribe fresh from sequence 0. diff --git a/server/lib/coflux/orchestration/workspaces.ex b/server/lib/coflux/orchestration/workspaces.ex index 8ec96f09..4c606c8d 100644 --- a/server/lib/coflux/orchestration/workspaces.ex +++ b/server/lib/coflux/orchestration/workspaces.ex @@ -572,7 +572,7 @@ defmodule Coflux.Orchestration.Workspaces do |> apply_patch_field(patch, :modules) |> apply_patch_field(patch, :provides) |> apply_patch_field(patch, :accepts) - |> apply_patch_field(patch, :idle_timeout) + |> apply_patch_field(patch, :idle_timeout_ms) |> apply_launcher_patch(patch) end @@ -732,7 +732,7 @@ defmodule Coflux.Orchestration.Workspaces do provides_tag_set_id, accepts_tag_set_id, modules, - idle_timeout + idle_timeout_ms ) do launcher_hash = if launcher_id do @@ -766,7 +766,7 @@ defmodule Coflux.Orchestration.Workspaces do # Only part of the hash when set, so a definition without one keeps # the hash it had before the field existed. - idle_timeout_part = if idle_timeout, do: [Integer.to_string(idle_timeout)], else: [] + idle_timeout_part = if idle_timeout_ms, do: [Integer.to_string(idle_timeout_ms)], else: [] data = Enum.intersperse( @@ -787,7 +787,7 @@ defmodule Coflux.Orchestration.Workspaces do provides = Map.get(pool, :provides, %{}) accepts = Map.get(pool, :accepts, %{}) launcher = Map.get(pool, :launcher) - idle_timeout = Map.get(pool, :idle_timeout) + idle_timeout_ms = Map.get(pool, :idle_timeout_ms) launcher_id = if launcher do @@ -817,7 +817,7 @@ defmodule Coflux.Orchestration.Workspaces do provides_tag_set_id, accepts_tag_set_id, modules, - idle_timeout + idle_timeout_ms ) case query_one(db, "SELECT id FROM pool_definitions WHERE hash = ?1", {{:blob, hash}}) do @@ -831,7 +831,7 @@ defmodule Coflux.Orchestration.Workspaces do provides_tag_set_id: provides_tag_set_id, accepts_tag_set_id: accepts_tag_set_id, launcher_id: launcher_id, - idle_timeout: idle_timeout + idle_timeout_ms: idle_timeout_ms }) {:ok, _} = @@ -914,10 +914,10 @@ defmodule Coflux.Orchestration.Workspaces do defp get_pool_definition(db, pool_definition_id) do case query_one( db, - "SELECT launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout FROM pool_definitions WHERE id = ?1", + "SELECT launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout_ms FROM pool_definitions WHERE id = ?1", {pool_definition_id} ) do - {:ok, {launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout}} -> + {:ok, {launcher_id, provides_tag_set_id, accepts_tag_set_id, idle_timeout_ms}} -> provides = if provides_tag_set_id do case TagSets.get_tag_set(db, provides_tag_set_id) do @@ -959,7 +959,7 @@ defmodule Coflux.Orchestration.Workspaces do accepts: accepts, modules: modules, launcher: launcher, - idle_timeout: idle_timeout + idle_timeout_ms: idle_timeout_ms }} {:ok, nil} -> diff --git a/server/lib/coflux/run_view.ex b/server/lib/coflux/run_view.ex index a66e8cae..5ea89ed0 100644 --- a/server/lib/coflux/run_view.ex +++ b/server/lib/coflux/run_view.ex @@ -397,7 +397,7 @@ defmodule Coflux.RunView do group_limit: e.group_limit, retries: e.retries, recurrent: e.recurrent == true, - timeout: e.timeout, + timeout_ms: e.timeout_ms, created_at: e.created_at, arguments: nil, requires: e.requires, @@ -1122,7 +1122,7 @@ defmodule Coflux.RunView do group: Format.group(step), retries: Format.retries(step.retries), recurrent: step.recurrent, - timeout: step.timeout, + timeoutMs: step.timeout_ms, createdAt: step.created_at, arguments: Enum.map(step.arguments || [], &Format.value/1), requires: step.requires, diff --git a/server/lib/coflux/run_view/format.ex b/server/lib/coflux/run_view/format.ex index 6e23a2d5..c15d96bb 100644 --- a/server/lib/coflux/run_view/format.ex +++ b/server/lib/coflux/run_view/format.ex @@ -47,7 +47,7 @@ defmodule Coflux.RunView.Format do def cache_config(cache_config) do %{ params: cache_config.params, - maxAge: cache_config.max_age, + maxAgeMs: cache_config.max_age_ms, namespace: cache_config.namespace, version: cache_config.version } @@ -69,8 +69,8 @@ defmodule Coflux.RunView.Format do def retries(nil), do: nil - def retries(%{limit: limit, backoff_min: backoff_min, backoff_max: backoff_max}) do - %{limit: limit, backoffMin: backoff_min, backoffMax: backoff_max} + def retries(%{limit: limit, backoff_min_ms: backoff_min_ms, backoff_max_ms: backoff_max_ms}) do + %{limit: limit, backoffMinMs: backoff_min_ms, backoffMaxMs: backoff_max_ms} end def key(key, length \\ 10) diff --git a/server/lib/coflux/topics/manifests.ex b/server/lib/coflux/topics/manifests.ex index 1b9db5ce..796c5979 100644 --- a/server/lib/coflux/topics/manifests.ex +++ b/server/lib/coflux/topics/manifests.ex @@ -85,7 +85,7 @@ defmodule Coflux.Topics.Manifests.Model do waitFor: workflow.wait_for, cache: build_cache(workflow.cache), defer: build_defer(workflow.defer), - delay: workflow.delay, + delayMs: workflow.delay_ms, retries: build_retries(workflow.retries), requires: workflow.requires, concurrency: build_concurrency(workflow[:concurrency]) @@ -97,7 +97,7 @@ defmodule Coflux.Topics.Manifests.Model do defp build_cache(cache) do %{ params: cache.params, - maxAge: cache.max_age, + maxAgeMs: cache.max_age_ms, namespace: cache.namespace, version: cache.version } @@ -121,8 +121,8 @@ defmodule Coflux.Topics.Manifests.Model do defp build_retries(retries) do %{ limit: retries.limit, - backoffMin: retries.backoff_min, - backoffMax: retries.backoff_max + backoffMinMs: retries.backoff_min_ms, + backoffMaxMs: retries.backoff_max_ms } end end diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index 0a22bc0d..0cdb483d 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -46,7 +46,7 @@ defmodule Coflux.Topics.Pool do launcher: if(pool.launcher, do: build_launcher(pool.launcher)), state: to_string(Map.get(pool, :state, :active)) } - |> maybe_put(:idleTimeout, Map.get(pool, :idle_timeout)) + |> maybe_put(:idleTimeoutMs, Map.get(pool, :idle_timeout_ms)) end defp build_launcher(launcher) do diff --git a/server/lib/coflux/topics/workflow.ex b/server/lib/coflux/topics/workflow.ex index 25b6a3bc..667058fd 100644 --- a/server/lib/coflux/topics/workflow.ex +++ b/server/lib/coflux/topics/workflow.ex @@ -177,7 +177,7 @@ defmodule Coflux.Topics.Workflow.Model do if cache do %{ params: cache.params, - maxAge: cache.max_age, + maxAgeMs: cache.max_age_ms, namespace: cache.namespace, version: cache.version } @@ -194,8 +194,8 @@ defmodule Coflux.Topics.Workflow.Model do if retries do %{ limit: retries.limit, - backoffMin: retries.backoff_min, - backoffMax: retries.backoff_max + backoffMinMs: retries.backoff_min_ms, + backoffMaxMs: retries.backoff_max_ms } end end @@ -206,10 +206,10 @@ defmodule Coflux.Topics.Workflow.Model do waitFor: workflow.wait_for, cache: build_cache_configuration(workflow.cache), defer: build_defer_configuration(workflow.defer), - delay: workflow.delay, + delayMs: workflow.delay_ms, retries: build_retries_configuration(workflow.retries), recurrent: workflow.recurrent, - timeout: workflow.timeout, + timeoutMs: workflow.timeout_ms, requires: workflow.requires, memo: workflow.memo, streams: build_streams_configuration(workflow[:streams]), diff --git a/server/priv/migrations/orchestration/6.sql b/server/priv/migrations/orchestration/6.sql index b25c5ef2..54734bc8 100644 --- a/server/priv/migrations/orchestration/6.sql +++ b/server/priv/migrations/orchestration/6.sql @@ -135,9 +135,9 @@ ALTER TABLE assignments ADD COLUMN catalog_sequence INTEGER; ALTER TABLE runs ADD COLUMN catalog_sequence INTEGER; ALTER TABLE executions ADD COLUMN catalog_sequence INTEGER; --- How long a pool keeps an idle worker before stopping it, in seconds. +-- How long a pool keeps an idle worker before stopping it, in milliseconds. -- NULL leaves it to the scheduler's default. -ALTER TABLE pool_definitions ADD COLUMN idle_timeout INTEGER; +ALTER TABLE pool_definitions ADD COLUMN idle_timeout_ms INTEGER; -- Tokens have moved to the admin store, which isn't rotated, so a -- principal names its token by external id rather than by a row in this @@ -160,3 +160,28 @@ WHERE p.user_external_id IS NOT NULL OR t.external_id IS NOT NULL; DROP TABLE principals; ALTER TABLE principals_new RENAME TO principals; + +-- Durations carry their unit in the name. +-- +-- Every duration in this database is, and always has been, an integer +-- number of milliseconds, but only the columns added most recently said +-- so (`streams_timeout_ms`, `timeout_ms`). The rest read as bare +-- `timeout` / `delay` / `max_age`, which is how the pool idle timeout +-- came to be added in seconds without anyone noticing the mismatch. +-- +-- Renames only: no value changes, and no column here is referenced by a +-- view, trigger or index. +ALTER TABLE cache_configs RENAME COLUMN max_age TO max_age_ms; + +ALTER TABLE workflows RENAME COLUMN delay TO delay_ms; +ALTER TABLE workflows RENAME COLUMN timeout TO timeout_ms; +ALTER TABLE workflows RENAME COLUMN retry_backoff_min TO retry_backoff_min_ms; +ALTER TABLE workflows RENAME COLUMN retry_backoff_max TO retry_backoff_max_ms; + +ALTER TABLE steps RENAME COLUMN delay TO delay_ms; +ALTER TABLE steps RENAME COLUMN timeout TO timeout_ms; +ALTER TABLE steps RENAME COLUMN retry_backoff_min TO retry_backoff_min_ms; +ALTER TABLE steps RENAME COLUMN retry_backoff_max TO retry_backoff_max_ms; + +ALTER TABLE sessions RENAME COLUMN activation_timeout TO activation_timeout_ms; +ALTER TABLE sessions RENAME COLUMN reconnection_timeout TO reconnection_timeout_ms; diff --git a/server/test/coflux/catalog_test.exs b/server/test/coflux/catalog_test.exs index f4794f29..e3ed8f12 100644 --- a/server/test/coflux/catalog_test.exs +++ b/server/test/coflux/catalog_test.exs @@ -328,7 +328,7 @@ defmodule Coflux.CatalogTest do Sqlite3.execute(db, """ INSERT INTO steps ( id, number, run_id, module, target, type, priority, wait_for, - retry_limit, retry_backoff_min, retry_backoff_max, created_at + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, created_at ) VALUES (#{id}, #{number}, #{run_id}, 'module', 'target', 0, 0, 0, 0, 0, 0, 0) """) diff --git a/server/test/coflux/checkpoints_test.exs b/server/test/coflux/checkpoints_test.exs index 9344bacc..90c80b99 100644 --- a/server/test/coflux/checkpoints_test.exs +++ b/server/test/coflux/checkpoints_test.exs @@ -306,7 +306,7 @@ defmodule Coflux.CheckpointsTest do Sqlite3.execute(db, """ INSERT INTO steps ( id, number, run_id, module, target, type, priority, wait_for, - retry_limit, retry_backoff_min, retry_backoff_max, created_at + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, created_at ) VALUES (#{id}, #{number}, #{run_id}, 'module', 'target', 0, 0, 0, 0, 0, 0, 0) """) diff --git a/server/test/coflux/concurrency_permits_test.exs b/server/test/coflux/concurrency_permits_test.exs index 4b399ab9..73886a9b 100644 --- a/server/test/coflux/concurrency_permits_test.exs +++ b/server/test/coflux/concurrency_permits_test.exs @@ -137,7 +137,7 @@ defmodule Coflux.ConcurrencyPermitsTest do Sqlite3.execute(db, """ INSERT INTO steps ( id, number, run_id, module, target, type, priority, wait_for, - retry_limit, retry_backoff_min, retry_backoff_max, + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, concurrency_key, concurrency_limit, group_key, group_limit, created_at ) VALUES ( diff --git a/server/test/coflux/events_test.exs b/server/test/coflux/events_test.exs index 92edf801..ed8f7787 100644 --- a/server/test/coflux/events_test.exs +++ b/server/test/coflux/events_test.exs @@ -27,10 +27,10 @@ defmodule Coflux.EventsTest do wait_for: [], cache: nil, defer: nil, - delay: 0, + delay_ms: 0, retries: nil, recurrent: false, - timeout: 0, + timeout_ms: 0, requires: %{}, memo: false, streams: nil, @@ -198,12 +198,12 @@ defmodule Coflux.EventsTest do :ok = Orchestration.register_manifests(project, ws, %{ - "test" => %{"main" => %{@workflow | delay: 5}, "other" => @workflow} + "test" => %{"main" => %{@workflow | delay_ms: 5}, "other" => @workflow} }) {subs, models} = check(project, subs) assert map_size(models[{:workflow, "test", "main", ws}].runs) == 2 - assert models[{:workflow, "test", "main", ws}].workflow.delay == 5 + assert models[{:workflow, "test", "main", ws}].workflow.delay_ms == 5 assert models[{:targets, ws}]["test"]["other"] == {:workflow, nil} :ok = Orchestration.archive_module(project, ws, "test") diff --git a/server/test/coflux/run_view_test.exs b/server/test/coflux/run_view_test.exs index 288c2021..dadf2f54 100644 --- a/server/test/coflux/run_view_test.exs +++ b/server/test/coflux/run_view_test.exs @@ -80,7 +80,7 @@ defmodule Coflux.RunViewTest do group_limit: 0, retries: nil, recurrent: false, - timeout: 0, + timeout_ms: 0, created_at: number * 10, requires: %{} } diff --git a/server/test/coflux/streams_test.exs b/server/test/coflux/streams_test.exs index 9378776c..44560606 100644 --- a/server/test/coflux/streams_test.exs +++ b/server/test/coflux/streams_test.exs @@ -238,7 +238,7 @@ defmodule Coflux.StreamsTest do Sqlite3.execute(db, """ INSERT INTO steps ( id, number, run_id, module, target, type, priority, wait_for, - retry_limit, retry_backoff_min, retry_backoff_max, created_at + retry_limit, retry_backoff_min_ms, retry_backoff_max_ms, created_at ) VALUES (#{id}, #{number}, #{run_id}, 'module', 'target', 0, 0, 0, 0, 0, 0, 0) """) diff --git a/server/test/coflux/topics/workflow_model_test.exs b/server/test/coflux/topics/workflow_model_test.exs index e8f3ae0d..c5c624c3 100644 --- a/server/test/coflux/topics/workflow_model_test.exs +++ b/server/test/coflux/topics/workflow_model_test.exs @@ -19,10 +19,10 @@ defmodule Coflux.Topics.WorkflowModelTest do wait_for: [], cache: nil, defer: nil, - delay: 5, - retries: %{limit: 2, backoff_min: 0, backoff_max: 0}, + delay_ms: 5, + retries: %{limit: 2, backoff_min_ms: 0, backoff_max_ms: 0}, recurrent: false, - timeout: 0, + timeout_ms: 0, requires: %{}, memo: false, streams: nil, @@ -105,8 +105,8 @@ defmodule Coflux.Topics.WorkflowModelTest do value = Model.project(fold([registered()])) assert value.parameters == [%{name: "x", default: nil, annotation: nil}] assert value.instruction == "Do it" - assert value.configuration.delay == 5 - assert value.configuration.retries == %{limit: 2, backoffMin: 0, backoffMax: 0} + assert value.configuration.delayMs == 5 + assert value.configuration.retries == %{limit: 2, backoffMinMs: 0, backoffMaxMs: 0} assert value.configuration.requires == %{} end diff --git a/tests/support/manifest.py b/tests/support/manifest.py index cb3f8db1..acdfb685 100644 --- a/tests/support/manifest.py +++ b/tests/support/manifest.py @@ -32,7 +32,7 @@ def _target( if memo is not None: target["memo"] = memo if delay is not None: - target["delay"] = delay + target["delay_ms"] = delay if recurrent: target["recurrent"] = True if wait_for is not None: @@ -40,7 +40,7 @@ def _target( if requires is not None: target["requires"] = requires if timeout is not None: - target["timeout"] = timeout + target["timeout_ms"] = timeout if streams is not None: target["streams"] = streams if concurrency is not None: diff --git a/tests/support/protocol.py b/tests/support/protocol.py index 5acb36ab..58e03adf 100644 --- a/tests/support/protocol.py +++ b/tests/support/protocol.py @@ -88,7 +88,7 @@ def submit_execution_request( if group_id is not None: params["group_id"] = group_id if delay is not None: - params["delay"] = delay + params["delay_ms"] = delay if retries is not None: params["retries"] = retries if requires is not None: @@ -102,7 +102,7 @@ def submit_execution_request( if wait_for is not None: params["wait_for"] = wait_for if timeout: - params["timeout"] = timeout + params["timeout_ms"] = timeout return {"id": request_id, "method": "submit_execution", "params": params} diff --git a/tests/test_concurrency_limits.py b/tests/test_concurrency_limits.py index c450781e..2029cebc 100644 --- a/tests/test_concurrency_limits.py +++ b/tests/test_concurrency_limits.py @@ -546,7 +546,7 @@ def test_cache_hit_takes_no_permit(worker): resp = ctx.submit("test", "main") ex0 = ctx.executor.next_execute() - cache = {"params": True, "max_age": None, "namespace": None, "version": None} + cache = {"params": True, "max_age_ms": None, "namespace": None, "version": None} config = {"limit": 1} # Populate the cache, and let the execution go. diff --git a/tests/test_pools.py b/tests/test_pools.py index 4bb4c4a8..573251cd 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -306,17 +306,17 @@ def test_idle_timeout_is_a_pool_field(self, pool_env, tmp_path): level, and zero is a value rather than an absence.""" host = pool_env["host"] targets = [workflow("test", "my_workflow")] - _setup_pool(pool_env, targets, pool_name="idle-pool", idle_timeout=300) + _setup_pool(pool_env, targets, pool_name="idle-pool", idle_timeout="5m") pool = cli.pools_get("idle-pool", host=host) - assert pool["idleTimeout"] == 300 - assert "idleTimeout" not in pool["launcher"] + assert pool["idleTimeoutMs"] == 300_000 + assert "idleTimeoutMs" not in pool["launcher"] exported = cli.pools_export(host=host) - assert "idle_timeout = 300\n" in exported + assert 'idle_timeout = "5m"\n' in exported - cli.pools_update("idle-pool", idle_timeout=0, host=host) - assert cli.pools_get("idle-pool", host=host)["idleTimeout"] == 0 + cli.pools_update("idle-pool", idle_timeout="0s", host=host) + assert cli.pools_get("idle-pool", host=host)["idleTimeoutMs"] == 0 cli._coflux( "pools", @@ -327,12 +327,12 @@ def test_idle_timeout_is_a_pool_field(self, pool_env, tmp_path): host=host, output=None, ) - assert "idleTimeout" not in cli.pools_get("idle-pool", host=host) + assert "idleTimeoutMs" not in cli.pools_get("idle-pool", host=host) path = tmp_path / "pools.toml" path.write_text(exported) cli.pools_import(path, host=host) - assert cli.pools_get("idle-pool", host=host)["idleTimeout"] == 300 + assert cli.pools_get("idle-pool", host=host)["idleTimeoutMs"] == 300_000 def test_idle_timeout_keeps_worker_warm(self, pool_env): """A worker with an idle timeout outlives the gap between runs, so @@ -340,7 +340,7 @@ def test_idle_timeout_keeps_worker_warm(self, pool_env): host = pool_env["host"] executor = pool_env["executor"] targets = [workflow("test", "greet", parameters=["name"])] - _setup_pool(pool_env, targets, pool_name="warm-pool", idle_timeout=60) + _setup_pool(pool_env, targets, pool_name="warm-pool", idle_timeout="60s") resp = cli.submit("test/greet", '"one"', host=host) executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) diff --git a/tests/test_topics.py b/tests/test_topics.py index 6d12c025..3f1932bf 100644 --- a/tests/test_topics.py +++ b/tests/test_topics.py @@ -184,10 +184,10 @@ def topic(): "waitFor": [], "cache": None, "defer": None, - "delay": 5, - "retries": {"limit": 2, "backoffMin": 0, "backoffMax": 0}, + "delayMs": 5, + "retries": {"limit": 2, "backoffMinMs": 0, "backoffMaxMs": 0}, "recurrent": False, - "timeout": 0, + "timeoutMs": 0, "requires": {}, "memo": False, "streams": None, @@ -294,8 +294,8 @@ def topic(): "waitFor": [], "cache": None, "defer": None, - "delay": 3, - "retries": {"limit": 1, "backoffMin": 0, "backoffMax": 0}, + "delayMs": 3, + "retries": {"limit": 1, "backoffMinMs": 0, "backoffMaxMs": 0}, "requires": {}, "concurrency": None, } From d2f6177d42b74ba458ad849fa4c46c0143cf5182 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 15:40:03 +0100 Subject: [PATCH 15/21] Clean up after tests --- tests/conftest.py | 19 +++++++++++++++---- tests/test_auth.py | 6 ++---- tests/test_tokens.py | 6 ++---- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b1d4e2a5..bb8d6535 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,10 @@ import json -import tempfile import time import uuid from contextlib import contextmanager import pytest +from _pytest.tmpdir import TempPathFactory from support import cli from support.helpers import ( ADAPTER_SCRIPT, @@ -229,10 +229,21 @@ def run(self, module, target, *arguments): def pytest_configure(config): - """Start a shared test server (runs on the controller and in non-xdist mode).""" + """Start a shared test server (runs on the controller and in non-xdist mode). + + The data directory comes from pytest's temp-dir machinery rather than + `tempfile.mkdtemp`, so that something owns it: the last few runs are + kept, which leaves a failed run's server data there to inspect, and + older ones are collected. A directory made with `mkdtemp` is never + anyone's to remove, so it simply accumulates. + + A hook rather than a fixture because `pytest_configure_node` has to + hand the port to each xdist worker before any test runs — hence asking + `TempPathFactory` for itself instead of taking `tmp_path_factory`. + """ if not hasattr(config, "workerinput"): - data_dir = tempfile.mkdtemp(prefix="coflux-test-server-") - srv = ManagedServer(data_dir) + factory = TempPathFactory.from_config(config, _ispytest=True) + srv = ManagedServer(str(factory.mktemp("shared-server"))) srv.start() config._server = srv diff --git a/tests/test_auth.py b/tests/test_auth.py index 3a1b1905..75ef7121 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -6,7 +6,6 @@ """ import json -import tempfile import time import urllib.error import urllib.request @@ -43,11 +42,10 @@ def jwks_server(keypair): @pytest.fixture(scope="module") -def auth_server(jwks_server): +def auth_server(jwks_server, tmp_path_factory): """A Coflux server with authentication enabled.""" - data_dir = tempfile.mkdtemp(prefix="coflux-test-auth-") srv = ManagedServer( - data_dir, + str(tmp_path_factory.mktemp("auth-server")), extra_env={ "COFLUX_REQUIRE_AUTH": "true", "COFLUX_STUDIO_TEAMS": TEAM_ID, diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 42fe8e20..67c7150c 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -5,7 +5,6 @@ """ import json -import tempfile import urllib.error import urllib.request import uuid @@ -16,11 +15,10 @@ @pytest.fixture(scope="module") -def token_server(): +def token_server(tmp_path_factory): """A server with authentication on and a secret to sign tokens with.""" - data_dir = tempfile.mkdtemp(prefix="coflux-test-tokens-") srv = ManagedServer( - data_dir, + str(tmp_path_factory.mktemp("token-server")), extra_env={ "COFLUX_REQUIRE_AUTH": "true", "COFLUX_SECRET": "test-secret-for-service-tokens", From 3cbef8f7fa59a5701bd2d101394765e892896388 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 16:22:29 +0100 Subject: [PATCH 16/21] Improve API error messages --- cli/internal/api/client.go | 105 ++++-- server/lib/coflux/handlers/api.ex | 577 +++++++++++++----------------- 2 files changed, 335 insertions(+), 347 deletions(-) diff --git a/cli/internal/api/client.go b/cli/internal/api/client.go index 83da6787..e56232cb 100644 --- a/cli/internal/api/client.go +++ b/cli/internal/api/client.go @@ -10,10 +10,85 @@ import ( "maps" "net/http" "net/url" + "sort" + "strings" "github.com/bitroot/coflux/cli/internal/version" ) +// reasonText renders the server's validation vocabulary. Anything not +// listed is printed as-is, so a new reason degrades to its own name +// rather than disappearing. +var reasonText = map[string]string{ + "required": "required", + "invalid": "invalid", + "invalid_name": "not a valid name", + "malformed": "malformed", + "too_long": "too long", + "too_many": "too many", + "out_of_range": "out of range", + "unknown_value": "not one of the accepted values", + "exclusive": "cannot be combined with the other field given", + "reserved": "uses a reserved name", +} + +// flattenDetails turns the server's nested `details` into one line per +// field: {"pools": {"mypool": {"launcher": {"region": "required"}}}} +// becomes "pools.mypool.launcher.region: required". +func flattenDetails(prefix string, details map[string]any) []string { + var lines []string + for key, value := range details { + path := key + if prefix != "" { + path = prefix + "." + key + } + switch v := value.(type) { + case map[string]any: + lines = append(lines, flattenDetails(path, v)...) + case string: + text, ok := reasonText[v] + if !ok { + text = v + } + lines = append(lines, fmt.Sprintf("%s: %s", path, text)) + default: + lines = append(lines, fmt.Sprintf("%s: %v", path, v)) + } + } + return lines +} + +// apiError renders an error response. The server names the field it +// rejected and why, so print that rather than the raw JSON: a bad region +// in a pool's launcher reads as +// +// bad request +// pools.mypool.launcher.region: required +func apiError(statusCode int, body []byte) error { + var parsed struct { + Error string `json:"error"` + Message string `json:"message"` + Details map[string]any `json:"details"` + } + if json.Unmarshal(body, &parsed) != nil || parsed.Error == "" { + return fmt.Errorf("HTTP %d: %s", statusCode, string(body)) + } + + // The error code is left as-is: it is a stable identifier that scripts + // and tests match on, and the detail lines below carry the prose. + headline := parsed.Error + if parsed.Message != "" { + headline += ": " + parsed.Message + } + + lines := flattenDetails("", parsed.Details) + if len(lines) == 0 { + return errors.New(headline) + } + sort.Strings(lines) + return fmt.Errorf("%s\n %s", headline, strings.Join(lines, "\n ")) +} + // Client provides HTTP API access to the Coflux server type Client struct { baseURL string @@ -309,7 +384,7 @@ func (c *Client) UpdatePools(ctx context.Context, workspaceID string, pools map[ if err := checkVersionMismatch(resp.StatusCode, respBody); err != nil { return err } - return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody)) + return apiError(resp.StatusCode, respBody) } return nil @@ -595,19 +670,7 @@ func (c *Client) get(ctx context.Context, path string, params url.Values, result if err := checkVersionMismatch(resp.StatusCode, respBody); err != nil { return err } - var errResp struct { - Error string `json:"error"` - Message string `json:"message"` - Details any `json:"details"` - } - if json.Unmarshal(respBody, &errResp) == nil && errResp.Error != "" { - if errResp.Details != nil { - detailsJSON, _ := json.Marshal(errResp.Details) - return fmt.Errorf("%s: %s (details: %s)", errResp.Error, errResp.Message, string(detailsJSON)) - } - return fmt.Errorf("%s: %s", errResp.Error, errResp.Message) - } - return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody)) + return apiError(resp.StatusCode, respBody) } if result != nil && len(respBody) > 0 { @@ -648,19 +711,7 @@ func (c *Client) post(ctx context.Context, path string, body any, result any) (h if err := checkVersionMismatch(resp.StatusCode, respBody); err != nil { return nil, err } - var errResp struct { - Error string `json:"error"` - Message string `json:"message"` - Details any `json:"details"` - } - if json.Unmarshal(respBody, &errResp) == nil && errResp.Error != "" { - if errResp.Details != nil { - detailsJSON, _ := json.Marshal(errResp.Details) - return nil, fmt.Errorf("%s: %s (details: %s)", errResp.Error, errResp.Message, string(detailsJSON)) - } - return nil, fmt.Errorf("%s: %s", errResp.Error, errResp.Message) - } - return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(respBody)) + return nil, apiError(resp.StatusCode, respBody) } if result != nil && len(respBody) > 0 { diff --git a/server/lib/coflux/handlers/api.ex b/server/lib/coflux/handlers/api.ex index e623bef6..38dd8129 100644 --- a/server/lib/coflux/handlers/api.ex +++ b/server/lib/coflux/handlers/api.ex @@ -7,6 +7,7 @@ defmodule Coflux.Handlers.Api do @max_parameters 20 @ecs_launch_types ["FARGATE", "EC2", "EXTERNAL"] + @pull_policies ["Always", "Never", "IfNotPresent"] # An IAM role ARN: any partition, a 12-digit account, and a name that # may sit under a path. @@ -1191,7 +1192,7 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(result, key, value)}} {:error, error} -> - {:halt, {:error, error}} + {:halt, {:error, %{key => error}}} end end) @@ -1200,295 +1201,274 @@ defmodule Coflux.Handlers.Api do end end - defp parse_docker_launcher(value) do - image = Map.get(value, "image") - docker_host = Map.get(value, "dockerHost") - network_mode = Map.get(value, "networkMode") - - cond do - not is_binary(image) or String.length(image) > 200 -> - {:error, :invalid} - - not is_nil(docker_host) and (not is_binary(docker_host) or String.length(docker_host) > 200) -> - {:error, :invalid} - - not is_nil(network_mode) and - (not is_binary(network_mode) or String.length(network_mode) > 200) -> - {:error, :invalid} - - true -> - launcher = %{type: :docker, image: image} + # --- Field validation ------------------------------------------------ + # + # Each check returns `:ok`, or `{:error, %{field => reason}}` naming the + # field it rejected. `merge_error/3` folds those into a dotted path, so a + # bad region in a pool's launcher reaches the client as + # `pools.my-pool.launcher.region` rather than a bare `invalid`, which + # says only that something somewhere in the request was wrong. + # + # Reasons are a small fixed vocabulary, so a client can render them + # without parsing prose: `:required`, `:invalid` (wrong type or shape), + # `:too_long`, `:too_many`, `:malformed`, `:out_of_range`, + # `:unknown_value`, `:exclusive`, `:reserved`. - launcher = - if docker_host, do: Map.put(launcher, :docker_host, docker_host), else: launcher - - launcher = - if network_mode, do: Map.put(launcher, :network_mode, network_mode), else: launcher - - {:ok, launcher} + defp required(value, field, kind, opts) do + case Map.get(value, field) do + nil -> {:error, %{field => :required}} + field_value -> check_field(field_value, field, kind, opts) end end - defp parse_process_launcher(value) do - directory = Map.get(value, "directory") - - cond do - not is_binary(directory) or String.length(directory) > 500 -> - {:error, :invalid} - - true -> - {:ok, %{type: :process, directory: directory}} + defp optional(value, field, kind, opts \\ []) do + case Map.get(value, field) do + nil -> :ok + field_value -> check_field(field_value, field, kind, opts) end end - defp parse_kubernetes_launcher(value) do - image = Map.get(value, "image") - namespace = Map.get(value, "namespace") - service_account = Map.get(value, "serviceAccount") - api_server = Map.get(value, "apiServer") - token_secret = Map.get(value, "tokenSecret") - ca_cert = Map.get(value, "caCert") - insecure = Map.get(value, "insecure") - image_pull_policy = Map.get(value, "imagePullPolicy") - node_selector = Map.get(value, "nodeSelector") - tolerations = Map.get(value, "tolerations") - image_pull_secrets = Map.get(value, "imagePullSecrets") - host_aliases = Map.get(value, "hostAliases") - resources = Map.get(value, "resources") - labels = Map.get(value, "labels") - annotations = Map.get(value, "annotations") - active_deadline_seconds = Map.get(value, "activeDeadlineSeconds") - volumes = Map.get(value, "volumes") - volume_mounts = Map.get(value, "volumeMounts") - - valid_pull_policies = ["Always", "Never", "IfNotPresent"] - + defp check_field(value, field, :string, opts) do cond do - not is_binary(image) or String.length(image) > 200 -> - {:error, :invalid} - - not is_nil(namespace) and (not is_binary(namespace) or String.length(namespace) > 253) -> - {:error, :invalid} - - not is_nil(service_account) and - (not is_binary(service_account) or String.length(service_account) > 253) -> - {:error, :invalid} - - not is_nil(api_server) and (not is_binary(api_server) or String.length(api_server) > 500) -> - {:error, :invalid} - - not is_nil(token_secret) and not Coflux.Admin.Secrets.valid_name?(token_secret) -> - {:error, :invalid} - - not is_nil(ca_cert) and not is_binary(ca_cert) -> - {:error, :invalid} - - not is_nil(insecure) and not is_boolean(insecure) -> - {:error, :invalid} - - not is_nil(image_pull_policy) and image_pull_policy not in valid_pull_policies -> - {:error, :invalid} - - not is_nil(node_selector) and not is_map(node_selector) -> - {:error, :invalid} + not is_binary(value) -> + {:error, %{field => :invalid}} - not is_nil(tolerations) and not is_list(tolerations) -> - {:error, :invalid} - - not is_nil(image_pull_secrets) and - (not is_list(image_pull_secrets) or - Enum.any?(image_pull_secrets, &(not is_binary(&1)))) -> - {:error, :invalid} - - not is_nil(host_aliases) and not is_list(host_aliases) -> - {:error, :invalid} + opts[:non_empty] && value == "" -> + {:error, %{field => :required}} - not is_nil(resources) and not is_map(resources) -> - {:error, :invalid} + opts[:max] && String.length(value) > opts[:max] -> + {:error, %{field => :too_long}} - not is_nil(labels) and - (not is_map(labels) or - Enum.any?(labels, fn {k, v} -> not is_binary(k) or not is_binary(v) end)) -> - {:error, :invalid} + opts[:match] && not Regex.match?(opts[:match], value) -> + {:error, %{field => :malformed}} - not is_nil(annotations) and - (not is_map(annotations) or - Enum.any?(annotations, fn {k, v} -> not is_binary(k) or not is_binary(v) end)) -> - {:error, :invalid} + opts[:secret_name] && not Coflux.Admin.Secrets.valid_name?(value) -> + {:error, %{field => :malformed}} - not is_nil(active_deadline_seconds) and - (not is_integer(active_deadline_seconds) or active_deadline_seconds < 1) -> - {:error, :invalid} - - not is_nil(volumes) and not is_list(volumes) -> - {:error, :invalid} - - not is_nil(volume_mounts) and not is_list(volume_mounts) -> - {:error, :invalid} + opts[:prefixes] && not String.starts_with?(value, opts[:prefixes]) -> + {:error, %{field => :malformed}} true -> - launcher = %{type: :kubernetes, image: image} - - launcher = - if namespace, do: Map.put(launcher, :namespace, namespace), else: launcher - - launcher = - if service_account, - do: Map.put(launcher, :service_account, service_account), - else: launcher - - launcher = - if api_server, do: Map.put(launcher, :api_server, api_server), else: launcher - - launcher = - if token_secret, do: Map.put(launcher, :token_secret, token_secret), else: launcher - - launcher = if ca_cert, do: Map.put(launcher, :ca_cert, ca_cert), else: launcher - - launcher = - if insecure == true, do: Map.put(launcher, :insecure, true), else: launcher - - launcher = - if image_pull_policy, - do: Map.put(launcher, :image_pull_policy, image_pull_policy), - else: launcher - - launcher = - if node_selector, do: Map.put(launcher, :node_selector, node_selector), else: launcher - - launcher = - if tolerations, do: Map.put(launcher, :tolerations, tolerations), else: launcher - - launcher = - if image_pull_secrets, - do: Map.put(launcher, :image_pull_secrets, image_pull_secrets), - else: launcher - - launcher = - if host_aliases, - do: Map.put(launcher, :host_aliases, host_aliases), - else: launcher - - launcher = - if resources, do: Map.put(launcher, :resources, resources), else: launcher - - launcher = - if labels, do: Map.put(launcher, :labels, labels), else: launcher + :ok + end + end - launcher = - if annotations, do: Map.put(launcher, :annotations, annotations), else: launcher + defp check_field(value, field, :boolean, _opts) do + if is_boolean(value), do: :ok, else: {:error, %{field => :invalid}} + end - launcher = - if active_deadline_seconds, - do: Map.put(launcher, :active_deadline_seconds, active_deadline_seconds), - else: launcher + defp check_field(value, field, :integer, opts) do + cond do + not is_integer(value) -> {:error, %{field => :invalid}} + opts[:min] && value < opts[:min] -> {:error, %{field => :out_of_range}} + true -> :ok + end + end - launcher = - if volumes, do: Map.put(launcher, :volumes, volumes), else: launcher + defp check_field(value, field, :enum, opts) do + if value in opts[:values], do: :ok, else: {:error, %{field => :unknown_value}} + end - launcher = - if volume_mounts, do: Map.put(launcher, :volume_mounts, volume_mounts), else: launcher + defp check_field(value, field, :list, opts) do + cond do + not is_list(value) -> {:error, %{field => :invalid}} + opts[:max] && length(value) > opts[:max] -> {:error, %{field => :too_many}} + true -> :ok + end + end - {:ok, launcher} + defp check_field(value, field, :string_list, opts) do + cond do + not is_list(value) -> {:error, %{field => :invalid}} + opts[:min] && length(value) < opts[:min] -> {:error, %{field => :required}} + opts[:max] && length(value) > opts[:max] -> {:error, %{field => :too_many}} + Enum.any?(value, &(not is_binary(&1))) -> {:error, %{field => :invalid}} + true -> :ok end end - defp parse_ecs_launcher(value) do - cluster = Map.get(value, "cluster") - task_definition = Map.get(value, "taskDefinition") - region = Map.get(value, "region") - container_name = Map.get(value, "containerName") - launch_type = Map.get(value, "launchType") - capacity_provider = Map.get(value, "capacityProvider") - subnets = wrap_list(Map.get(value, "subnets")) - security_groups = wrap_list(Map.get(value, "securityGroups")) - assign_public_ip = Map.get(value, "assignPublicIp") - platform_version = Map.get(value, "platformVersion") - credentials_secret = Map.get(value, "credentialsSecret") - role_arn = Map.get(value, "roleArn") - role_external_id = Map.get(value, "roleExternalId") - endpoint = Map.get(value, "endpoint") + # Subnet and security-group IDs: a non-empty list of non-empty strings. + defp check_field(value, field, :id_list, opts) do + if is_string_list?(value, opts[:max]), do: :ok, else: {:error, %{field => :invalid}} + end + defp check_field(value, field, :string_map, opts) do cond do - not is_binary(cluster) or cluster == "" or String.length(cluster) > 255 -> - {:error, :invalid} + not is_map(value) -> + {:error, %{field => :invalid}} - not is_binary(task_definition) or task_definition == "" or - String.length(task_definition) > 500 -> - {:error, :invalid} + Enum.any?(value, fn {k, v} -> not is_binary(k) or not is_binary(v) end) -> + {:error, %{field => :invalid}} - not is_binary(region) or not Regex.match?(~r/^[a-z0-9-]{1,30}$/, region) -> - {:error, :invalid} + opts[:reserved_prefix] && + Enum.any?(value, fn {k, _} -> String.starts_with?(k, opts[:reserved_prefix]) end) -> + {:error, %{field => :reserved}} - not is_nil(container_name) and - (not is_binary(container_name) or String.length(container_name) > 255) -> - {:error, :invalid} + true -> + :ok + end + end - not is_nil(launch_type) and launch_type not in @ecs_launch_types -> - {:error, :invalid} + # Environment variables set from named secrets: keys as for `env`, values + # naming a secret rather than holding one. + defp check_field(value, field, :secret_map, opts) do + cond do + not is_map(value) -> + {:error, %{field => :invalid}} - not is_nil(capacity_provider) and - (not is_binary(capacity_provider) or String.length(capacity_provider) > 255) -> - {:error, :invalid} + Enum.any?(value, fn {k, v} -> + not is_binary(k) or not Coflux.Admin.Secrets.valid_name?(v) + end) -> + {:error, %{field => :invalid}} - # A capacity provider strategy decides the launch type itself. - not is_nil(launch_type) and not is_nil(capacity_provider) -> - {:error, :invalid} + opts[:reserved_prefix] && + Enum.any?(value, fn {k, _} -> String.starts_with?(k, opts[:reserved_prefix]) end) -> + {:error, %{field => :reserved}} - not is_nil(subnets) and not is_string_list?(subnets, 16) -> - {:error, :invalid} + true -> + :ok + end + end - not is_nil(security_groups) and not is_string_list?(security_groups, 5) -> - {:error, :invalid} + defp check_field(value, field, :map, _opts) do + if is_map(value), do: :ok, else: {:error, %{field => :invalid}} + end - not is_nil(assign_public_ip) and not is_boolean(assign_public_ip) -> - {:error, :invalid} + # Two fields that may not be given together. Both are named, since + # either one is a candidate for removal and the reader can't tell which + # from one half of the pair. + defp exclusive(value, field, with_field) do + if is_nil(Map.get(value, field)) or is_nil(Map.get(value, with_field)) do + :ok + else + {:error, %{field => :exclusive, with_field => :exclusive}} + end + end - not is_nil(platform_version) and - (not is_binary(platform_version) or String.length(platform_version) > 50) -> - {:error, :invalid} + # A field that means nothing on its own: giving it makes the other + # required, so that is what the error names. + defp together(value, field, needs) do + if is_nil(Map.get(value, field)) or not is_nil(Map.get(value, needs)) do + :ok + else + {:error, %{needs => :required}} + end + end - not is_nil(credentials_secret) and - not Coflux.Admin.Secrets.valid_name?(credentials_secret) -> - {:error, :invalid} + defp parse_docker_launcher(value) do + with :ok <- required(value, "image", :string, max: 200), + :ok <- optional(value, "dockerHost", :string, max: 200), + :ok <- optional(value, "networkMode", :string, max: 200) do + launcher = + %{type: :docker, image: Map.get(value, "image")} + |> maybe_put_value(:docker_host, Map.get(value, "dockerHost")) + |> maybe_put_value(:network_mode, Map.get(value, "networkMode")) - not is_nil(role_arn) and - (not is_binary(role_arn) or String.length(role_arn) > 2048 or - not Regex.match?(@iam_role_arn_regex, role_arn)) -> - {:error, :invalid} + {:ok, launcher} + end + end - # An external ID is something to assume a role with, so it needs one. - not is_nil(role_external_id) and - (is_nil(role_arn) or not is_binary(role_external_id) or - not Regex.match?(@external_id_regex, role_external_id)) -> - {:error, :invalid} + defp parse_process_launcher(value) do + with :ok <- required(value, "directory", :string, max: 500) do + {:ok, %{type: :process, directory: Map.get(value, "directory")}} + end + end - not is_nil(endpoint) and - (not is_binary(endpoint) or String.length(endpoint) > 500 or - not String.starts_with?(endpoint, ["http://", "https://"])) -> - {:error, :invalid} + defp parse_kubernetes_launcher(value) do + with :ok <- required(value, "image", :string, max: 200), + :ok <- optional(value, "namespace", :string, max: 253), + :ok <- optional(value, "serviceAccount", :string, max: 253), + :ok <- optional(value, "apiServer", :string, max: 500), + :ok <- optional(value, "tokenSecret", :string, secret_name: true), + :ok <- optional(value, "caCert", :string), + :ok <- optional(value, "insecure", :boolean), + :ok <- optional(value, "imagePullPolicy", :enum, values: @pull_policies), + :ok <- optional(value, "nodeSelector", :map), + :ok <- optional(value, "tolerations", :list), + :ok <- optional(value, "imagePullSecrets", :string_list), + :ok <- optional(value, "hostAliases", :list), + :ok <- optional(value, "resources", :map), + :ok <- optional(value, "labels", :string_map), + :ok <- optional(value, "annotations", :string_map), + :ok <- optional(value, "activeDeadlineSeconds", :integer, min: 1), + :ok <- optional(value, "volumes", :list), + :ok <- optional(value, "volumeMounts", :list) do + launcher = + %{type: :kubernetes, image: Map.get(value, "image")} + |> maybe_put_value(:namespace, Map.get(value, "namespace")) + |> maybe_put_value(:service_account, Map.get(value, "serviceAccount")) + |> maybe_put_value(:api_server, Map.get(value, "apiServer")) + |> maybe_put_value(:token_secret, Map.get(value, "tokenSecret")) + |> maybe_put_value(:ca_cert, Map.get(value, "caCert")) + |> maybe_put_value(:insecure, if(Map.get(value, "insecure") == true, do: true)) + |> maybe_put_value(:image_pull_policy, Map.get(value, "imagePullPolicy")) + |> maybe_put_value(:node_selector, Map.get(value, "nodeSelector")) + |> maybe_put_value(:tolerations, Map.get(value, "tolerations")) + |> maybe_put_value(:image_pull_secrets, Map.get(value, "imagePullSecrets")) + |> maybe_put_value(:host_aliases, Map.get(value, "hostAliases")) + |> maybe_put_value(:resources, Map.get(value, "resources")) + |> maybe_put_value(:labels, Map.get(value, "labels")) + |> maybe_put_value(:annotations, Map.get(value, "annotations")) + |> maybe_put_value(:active_deadline_seconds, Map.get(value, "activeDeadlineSeconds")) + |> maybe_put_value(:volumes, Map.get(value, "volumes")) + |> maybe_put_value(:volume_mounts, Map.get(value, "volumeMounts")) + + {:ok, launcher} + end + end - true -> - launcher = - %{type: :ecs, cluster: cluster, task_definition: task_definition, region: region} - |> maybe_put_value(:container_name, container_name) - |> maybe_put_value(:launch_type, launch_type) - |> maybe_put_value(:capacity_provider, capacity_provider) - |> maybe_put_value(:subnets, subnets) - |> maybe_put_value(:security_groups, security_groups) - |> maybe_put_value(:assign_public_ip, if(assign_public_ip == true, do: true)) - |> maybe_put_value(:platform_version, platform_version) - |> maybe_put_value(:credentials_secret, credentials_secret) - |> maybe_put_value(:role_arn, role_arn) - |> maybe_put_value(:role_external_id, role_external_id) - |> maybe_put_value(:endpoint, endpoint) + defp parse_ecs_launcher(value) do + # A single ID is accepted where a list is expected, so `--set + # subnets=subnet-1` works without JSON. + value = + value + |> Map.replace_lazy("subnets", &wrap_list/1) + |> Map.replace_lazy("securityGroups", &wrap_list/1) + + with :ok <- required(value, "cluster", :string, non_empty: true, max: 255), + :ok <- required(value, "taskDefinition", :string, non_empty: true, max: 500), + :ok <- required(value, "region", :string, match: ~r/^[a-z0-9-]{1,30}$/), + :ok <- optional(value, "containerName", :string, max: 255), + :ok <- optional(value, "launchType", :enum, values: @ecs_launch_types), + :ok <- optional(value, "capacityProvider", :string, max: 255), + # A capacity provider strategy decides the launch type itself. + :ok <- exclusive(value, "capacityProvider", "launchType"), + :ok <- optional(value, "subnets", :id_list, max: 16), + :ok <- optional(value, "securityGroups", :id_list, max: 5), + :ok <- optional(value, "assignPublicIp", :boolean), + :ok <- optional(value, "platformVersion", :string, max: 50), + :ok <- optional(value, "credentialsSecret", :string, secret_name: true), + :ok <- optional(value, "roleArn", :string, max: 2048, match: @iam_role_arn_regex), + # An external ID is something to assume a role with, so it needs one. + :ok <- together(value, "roleExternalId", "roleArn"), + :ok <- optional(value, "roleExternalId", :string, match: @external_id_regex), + :ok <- optional(value, "endpoint", :string, max: 500, prefixes: ["http://", "https://"]) do + launcher = + %{ + type: :ecs, + cluster: Map.get(value, "cluster"), + task_definition: Map.get(value, "taskDefinition"), + region: Map.get(value, "region") + } + |> maybe_put_value(:container_name, Map.get(value, "containerName")) + |> maybe_put_value(:launch_type, Map.get(value, "launchType")) + |> maybe_put_value(:capacity_provider, Map.get(value, "capacityProvider")) + |> maybe_put_value(:subnets, Map.get(value, "subnets")) + |> maybe_put_value(:security_groups, Map.get(value, "securityGroups")) + |> maybe_put_value( + :assign_public_ip, + if(Map.get(value, "assignPublicIp") == true, do: true) + ) + |> maybe_put_value(:platform_version, Map.get(value, "platformVersion")) + |> maybe_put_value(:credentials_secret, Map.get(value, "credentialsSecret")) + |> maybe_put_value(:role_arn, Map.get(value, "roleArn")) + |> maybe_put_value(:role_external_id, Map.get(value, "roleExternalId")) + |> maybe_put_value(:endpoint, Map.get(value, "endpoint")) - {:ok, launcher} + {:ok, launcher} end end - # A single ID is accepted where a list is expected, so `--set - # subnets=subnet-1` works without JSON. defp wrap_list(value) when is_binary(value), do: [value] defp wrap_list(value), do: value @@ -1498,67 +1478,22 @@ defmodule Coflux.Handlers.Api do end defp parse_common_launcher_fields(launcher, value) do - server_host = Map.get(value, "serverHost") - server_secure = Map.get(value, "serverSecure") - adapter = Map.get(value, "adapter") - concurrency = Map.get(value, "concurrency") - env = Map.get(value, "env") - env_secrets = Map.get(value, "envSecrets") - - cond do - not is_nil(server_host) and (not is_binary(server_host) or String.length(server_host) > 200) -> - {:error, :invalid} - - not is_nil(server_secure) and not is_boolean(server_secure) -> - {:error, :invalid} - - not is_nil(adapter) and - (not is_list(adapter) or adapter == [] or - Enum.any?(adapter, &(not is_binary(&1)))) -> - {:error, :invalid} - - not is_nil(concurrency) and (not is_integer(concurrency) or concurrency < 1) -> - {:error, :invalid} + with :ok <- optional(value, "serverHost", :string, max: 200), + :ok <- optional(value, "serverSecure", :boolean), + :ok <- optional(value, "adapter", :string_list, min: 1), + :ok <- optional(value, "concurrency", :integer, min: 1), + :ok <- optional(value, "env", :string_map, reserved_prefix: "COFLUX_"), + :ok <- optional(value, "envSecrets", :secret_map, reserved_prefix: "COFLUX_") do + launcher = + launcher + |> maybe_put_value(:server_host, Map.get(value, "serverHost")) + |> maybe_put_value(:server_secure, Map.get(value, "serverSecure")) + |> maybe_put_value(:adapter, Map.get(value, "adapter")) + |> maybe_put_value(:concurrency, Map.get(value, "concurrency")) + |> maybe_put_value(:env, Map.get(value, "env")) + |> maybe_put_value(:env_secrets, Map.get(value, "envSecrets")) - not is_nil(env) and not is_map(env) -> - {:error, :invalid} - - not is_nil(env) and - Enum.any?(env, fn {k, v} -> - not is_binary(k) or not is_binary(v) or String.starts_with?(k, "COFLUX_") - end) -> - {:error, :invalid} - - not is_nil(env_secrets) and not is_map(env_secrets) -> - {:error, :invalid} - - not is_nil(env_secrets) and - Enum.any?(env_secrets, fn {k, v} -> - not is_binary(k) or String.starts_with?(k, "COFLUX_") or - not Coflux.Admin.Secrets.valid_name?(v) - end) -> - {:error, :invalid} - - true -> - launcher = - if server_host, do: Map.put(launcher, :server_host, server_host), else: launcher - - launcher = - if not is_nil(server_secure), - do: Map.put(launcher, :server_secure, server_secure), - else: launcher - - launcher = if adapter, do: Map.put(launcher, :adapter, adapter), else: launcher - - launcher = - if concurrency, do: Map.put(launcher, :concurrency, concurrency), else: launcher - - launcher = if env, do: Map.put(launcher, :env, env), else: launcher - - launcher = - if env_secrets, do: Map.put(launcher, :env_secrets, env_secrets), else: launcher - - {:ok, launcher} + {:ok, launcher} end end @@ -1741,7 +1676,7 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(result, target, parsed)}} {:error, error} -> - {:halt, {:error, error}} + {:halt, {:error, %{source => error}}} end :error -> @@ -1784,7 +1719,7 @@ defmodule Coflux.Handlers.Api do {:ok, field_value} -> case parser.(field_value) do {:ok, parsed} -> {:cont, {:ok, Map.put(result, target, parsed)}} - {:error, error} -> {:halt, {:error, error}} + {:error, error} -> {:halt, {:error, %{source => error}}} end :error -> @@ -1830,8 +1765,6 @@ defmodule Coflux.Handlers.Api do defp parse_launcher_patch(_), do: {:error, :invalid} defp parse_launcher_patch_fields(value, type) do - valid_pull_policies = ["Always", "Never", "IfNotPresent"] - # All possible launcher fields with their validators field_specs = [ {"image", &is_binary/1}, @@ -1844,7 +1777,7 @@ defmodule Coflux.Handlers.Api do {"tokenSecret", &Coflux.Admin.Secrets.valid_name?/1}, {"caCert", &is_binary/1}, {"insecure", &is_boolean/1}, - {"imagePullPolicy", &(&1 in valid_pull_policies)}, + {"imagePullPolicy", &(&1 in @pull_policies)}, {"nodeSelector", &is_map/1}, {"tolerations", &is_list/1}, {"imagePullSecrets", &is_list/1}, @@ -1954,7 +1887,7 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(acc, atom_key, processed_value)}} else - {:halt, {:error, :invalid}} + {:halt, {:error, %{json_key => :invalid}}} end :error -> @@ -1998,10 +1931,12 @@ defmodule Coflux.Handlers.Api do defp parse_asset_entries(value) do if is_list(value) && value != [] && length(value) <= @max_asset_entries do result = - Enum.reduce_while(value, {:ok, []}, fn entry, {:ok, entries} -> + value + |> Enum.with_index() + |> Enum.reduce_while({:ok, []}, fn {entry, index}, {:ok, entries} -> case parse_asset_entry(entry) do {:ok, entry} -> {:cont, {:ok, [entry | entries]}} - {:error, error} -> {:halt, {:error, error}} + {:error, error} -> {:halt, {:error, %{index => error}}} end end) @@ -2152,10 +2087,12 @@ defmodule Coflux.Handlers.Api do is_list(value) && length(value) <= @max_parameters -> with {:ok, backwards} <- - Enum.reduce_while(value, {:ok, []}, fn item, {:ok, result} -> + value + |> Enum.with_index() + |> Enum.reduce_while({:ok, []}, fn {item, index}, {:ok, result} -> case parse_integer(item) do {:ok, value} -> {:cont, {:ok, [value | result]}} - {:error, error} -> {:halt, {:error, error}} + {:error, error} -> {:halt, {:error, %{index => error}}} end end) do {:ok, Enum.reverse(backwards)} @@ -2396,10 +2333,10 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(result, workflow_name, parsed)}} {:error, error} -> - {:halt, {:error, error}} + {:halt, {:error, %{workflow_name => error}}} end else - {:halt, {:error, :invalid}} + {:halt, {:error, %{workflow_name => :invalid_name}}} end end) end @@ -2413,10 +2350,10 @@ defmodule Coflux.Handlers.Api do {:cont, {:ok, Map.put(result, module, parsed)}} {:error, error} -> - {:halt, {:error, error}} + {:halt, {:error, %{module => error}}} end else - {:halt, {:error, :invalid}} + {:halt, {:error, %{module => :invalid_name}}} end end) else From 4b235c30ef0c26248545d0f2ced98a35f8f96323 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 17:14:30 +0100 Subject: [PATCH 17/21] Capture logs from task --- docs/docs/pools.md | 13 +- server/CHANGELOG.md | 1 + server/lib/coflux/launchers/ecs.ex | 207 +++++++++++++++++++-- server/test/coflux/launchers/ecs_test.exs | 212 ++++++++++++++++++++++ 4 files changed, 412 insertions(+), 21 deletions(-) create mode 100644 server/test/coflux/launchers/ecs_test.exs diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 4294cd1b..9162490b 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -162,13 +162,18 @@ Whichever identity calls ECS - the role, or the credentials themselves when there's no role - needs `ecs:RunTask`, `ecs:DescribeTasks` and `ecs:StopTask` on the cluster, `ecs:DescribeTaskDefinition` unless `containerName` is set, and `iam:PassRole` for the roles the task -definition names. Credentials that assume a role need `sts:AssumeRole` on +definition names. `logs:GetLogEvents` on the log group is optional: with +it a failed worker carries the tail of its output, and without it the +worker is still reported, with less to go on. Credentials that assume a role need `sts:AssumeRole` on it, and the role's trust policy has to allow them to. ECS doesn't expose container output through its API, so a worker's log -tail isn't shown; a task that fails to start reports its reason in its -place. Give the task definition a log configuration (`awslogs`, say) to -see what workers print. +tail is read from CloudWatch instead. That needs the task definition to +use the `awslogs` driver with a stream prefix, and the identity calling +ECS to have `logs:GetLogEvents` on the log group. Where any of that is +missing the worker reports what ECS itself said about the stop, which for +a container that exited is little more than its exit code — so a task +definition without a log configuration is worth giving one. | Field | Description | |-------|-------------| diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 1d92f100..0d6422f1 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -2,6 +2,7 @@ Enhancements: +- A worker launched on ECS that fails now carries the tail of its log stream, read from CloudWatch, the way the Docker, Kubernetes and process launchers already carry the container's own output. It needs the task definition to use the `awslogs` driver and the launcher's identity to have `logs:GetLogEvents`; without either, the worker reports what ECS said about the stop — now including the container's own reason and exit code rather than only the task-level message, which for a container exit says nothing but that it exited. - The ECS launcher can assume an IAM role (`roleArn`, with `roleExternalId` where the trust policy asks for one) with whatever credentials it finds, so a server in one account can launch into another. The search of the server's surroundings now includes a web identity token (`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`, as EKS sets for a pod whose service account has a role). Credentials issued by STS are cached until shortly before they expire. Changes: diff --git a/server/lib/coflux/launchers/ecs.ex b/server/lib/coflux/launchers/ecs.ex index dd9898c0..0c044952 100644 --- a/server/lib/coflux/launchers/ecs.ex +++ b/server/lib/coflux/launchers/ecs.ex @@ -26,6 +26,12 @@ defmodule Coflux.EcsLauncher do alias Coflux.Launchers.AwsCredentials @api_target_prefix "AmazonEC2ContainerServiceV20141113" + @logs_target_prefix "Logs_20140328" + + # Matches the other launchers, so a failed worker reads the same however + # it was launched. + @log_tail_lines 20 + @log_max_bytes 1024 @reason_max_bytes 1024 # ECS limits `startedBy` to 36 characters. @@ -96,7 +102,7 @@ defmodule Coflux.EcsLauncher do ecs_request(conn, "DescribeTasks", %{"cluster" => cluster, "tasks" => [task_arn]}) do case body do %{"tasks" => [task | _]} -> - interpret_task(task, data) + interpret_task(task, data, conn) # Stopped tasks are only described for an hour or so afterwards; # one that has aged out has nothing left to say. @@ -116,15 +122,148 @@ defmodule Coflux.EcsLauncher do # --- Task state --- - defp interpret_task(%{"lastStatus" => "STOPPED"} = task, data) do + defp interpret_task(%{"lastStatus" => "STOPPED"} = task, data, conn) do error = stop_error(task, data[:container_name]) - logs = if error, do: stopped_reason(task) + logs = if error, do: failure_logs(task, data[:container_name], conn) {:ok, false, error, logs} end # Anything else - provisioning, pending, running, or on its way to # stopped - is a task that hasn't finished yet. - defp interpret_task(_task, _data), do: {:ok, true} + defp interpret_task(_task, _data, _conn), do: {:ok, true} + + # --- Failure diagnostics --- + + # What to show for a worker that died. The other launchers put the + # container's own output here; ECS doesn't serve that through its API, + # so this reads CloudWatch when the task definition logs there and the + # role is allowed to read it, and falls back to what ECS itself said. + # + # Every step is best-effort. A worker that died has already been + # recorded; a log fetch that fails must never turn into a failed poll, + # so this narrows to a string or nil and swallows anything else. + defp failure_logs(task, container_name, conn) do + fetch_log_tail(task, container_name, conn) || stop_detail(task, container_name) + rescue + _ -> stop_detail(task, container_name) + catch + _, _ -> stop_detail(task, container_name) + end + + # ECS's own account of the stop: the task-level reason (for a container + # exit this is only ever "Essential container in task exited"), plus + # whatever the container itself reported, which is where anything + # specific tends to be. + defp stop_detail(task, container_name) do + container = find_container(task["containers"], container_name) + + [ + stopped_reason(task), + container && container["reason"], + container && exit_code_detail(container) + ] + |> Enum.reject(&(&1 in [nil, ""])) + |> Enum.uniq() + |> case do + [] -> nil + parts -> truncate_bytes(Enum.join(parts, " - "), @reason_max_bytes) + end + end + + defp exit_code_detail(%{"exitCode" => code}) when is_integer(code), do: "exit code #{code}" + defp exit_code_detail(_container), do: nil + + # The tail of the container's log stream, when there is one to read. + # Returns nil for every reason it might not be readable - no log + # configuration, a driver other than awslogs, no stream prefix to build + # the name from, the role lacking `logs:GetLogEvents`, or the events + # simply not having been delivered yet. + defp fetch_log_tail(task, container_name, conn) do + with {:ok, options} <- awslogs_options(task, container_name, conn), + {:ok, group} <- fetch_option(options, "awslogs-group"), + {:ok, prefix} <- fetch_option(options, "awslogs-stream-prefix"), + {:ok, name} <- container_name_for_logs(task, container_name), + {:ok, task_id} <- task_id(task) do + region = options["awslogs-region"] || conn.region + stream = "#{prefix}/#{name}/#{task_id}" + + case logs_request(conn, region, "GetLogEvents", %{ + "logGroupName" => group, + "logStreamName" => stream, + "limit" => @log_tail_lines, + "startFromHead" => false + }) do + {:ok, %{"events" => events}} when is_list(events) -> format_events(events) + _ -> nil + end + else + _ -> nil + end + end + + defp format_events(events) do + events + |> Enum.map(&Map.get(&1, "message")) + |> Enum.filter(&is_binary/1) + |> Enum.join("\n") + |> String.trim() + |> case do + "" -> nil + text -> truncate_bytes(text, @log_max_bytes) + end + end + + # The log configuration is on the container definition, so this costs a + # DescribeTaskDefinition - only ever on the failure path, and only worth + # anything for the awslogs driver. + defp awslogs_options(%{"taskDefinitionArn" => arn}, container_name, conn) when is_binary(arn) do + case ecs_request(conn, "DescribeTaskDefinition", %{"taskDefinition" => arn}) do + {:ok, %{"taskDefinition" => %{"containerDefinitions" => containers}}} + when is_list(containers) -> + container = + Enum.find(containers, &(&1["name"] == container_name)) || List.first(containers) + + case container do + %{"logConfiguration" => %{"logDriver" => "awslogs", "options" => options}} + when is_map(options) -> + {:ok, options} + + _ -> + :error + end + + _ -> + :error + end + end + + defp awslogs_options(_task, _container_name, _conn), do: :error + + defp fetch_option(options, key) do + case Map.get(options, key) do + value when is_binary(value) and value != "" -> {:ok, value} + _ -> :error + end + end + + # The stream is named after the container the log configuration belongs + # to, which is the one the pool overrides, or the task's only one. + defp container_name_for_logs(_task, name) when is_binary(name) and name != "", do: {:ok, name} + + defp container_name_for_logs(%{"containers" => [%{"name" => name} | _]}, _name) + when is_binary(name), + do: {:ok, name} + + defp container_name_for_logs(_task, _name), do: :error + + defp task_id(%{"taskArn" => arn}) when is_binary(arn) do + case arn |> String.split("/") |> List.last() do + id when is_binary(id) and id != "" -> {:ok, id} + _ -> :error + end + end + + defp task_id(_task), do: :error # Returns nil for a task that stopped because it was asked to, or an # error code for one that didn't. @@ -153,15 +292,11 @@ defmodule Coflux.EcsLauncher do defp failed_to_start_error(_reason), do: "task_failed_to_start" defp container_exit_error(containers, container_name) when is_list(containers) do - container = - Enum.find(containers, &(&1["name"] == container_name)) || - Enum.find(containers, &is_integer(&1["exitCode"])) - - case container do - %{"reason" => reason} when is_binary(reason) -> + case find_container(containers, container_name) do + %{"reason" => reason} = container when is_binary(reason) -> if reason =~ "OutOfMemory", do: "oom_killed", else: exit_code_error(container) - %{} -> + %{} = container -> exit_code_error(container) nil -> @@ -171,6 +306,13 @@ defmodule Coflux.EcsLauncher do defp container_exit_error(_containers, _container_name), do: "container_exited" + defp find_container(containers, container_name) when is_list(containers) do + Enum.find(containers, &(&1["name"] == container_name)) || + Enum.find(containers, &is_integer(&1["exitCode"])) + end + + defp find_container(_containers, _container_name), do: nil + defp exit_code_error(%{"exitCode" => 0}), do: nil defp exit_code_error(%{"exitCode" => code}) when is_integer(code), do: "exit_code:#{code}" defp exit_code_error(_container), do: "container_exited" @@ -363,13 +505,17 @@ defmodule Coflux.EcsLauncher do AwsCredentials.resolve(static_credentials(config), region: region, role_arn: config[:role_arn], - external_id: config[:role_external_id] + external_id: config[:role_external_id], + req_options: Map.get(config, :req_options, []) ) do {:ok, %{ region: region, endpoint: config[:endpoint] || default_endpoint(region), - credentials: credentials + credentials: credentials, + # Extra options for `Req.request/1`, so a test can stub the API + # without reaching AWS. Never set in normal operation. + req_options: Map.get(config, :req_options, []) }} end end @@ -391,12 +537,38 @@ defmodule Coflux.EcsLauncher do # The ECS API is JSON 1.1 over HTTPS: every call is a POST to the # regional endpoint, and the header says which operation. defp ecs_request(conn, action, body) do + aws_json_request( + conn, + "ecs", + conn.region, + conn.endpoint, + "#{@api_target_prefix}.#{action}", + body + ) + end + + # CloudWatch Logs, for the tail of a failed worker. A separate service + # and endpoint, but the same JSON 1.1 shape and the same credentials. + # An `endpoint` override on the pool names an ECS endpoint, so it is + # deliberately not reused here. + defp logs_request(conn, region, action, body) do + aws_json_request( + conn, + "logs", + region, + "https://logs.#{region}.amazonaws.com", + "#{@logs_target_prefix}.#{action}", + body + ) + end + + defp aws_json_request(conn, service, region, endpoint, target, body) do credentials = conn.credentials sigv4 = [ - service: "ecs", - region: conn.region, + service: service, + region: region, access_key_id: credentials.access_key_id, secret_access_key: credentials.secret_access_key ] @@ -405,16 +577,17 @@ defmodule Coflux.EcsLauncher do request = [ method: :post, - url: conn.endpoint, + url: endpoint, headers: [ {"content-type", "application/x-amz-json-1.1"}, - {"x-amz-target", "#{@api_target_prefix}.#{action}"} + {"x-amz-target", target} ], body: Jason.encode!(body), aws_sigv4: sigv4, retry: false, decode_body: false ] + |> Keyword.merge(Map.get(conn, :req_options, [])) case Req.request(request) do {:ok, %{status: status, body: raw}} -> diff --git a/server/test/coflux/launchers/ecs_test.exs b/server/test/coflux/launchers/ecs_test.exs new file mode 100644 index 00000000..e6968a0b --- /dev/null +++ b/server/test/coflux/launchers/ecs_test.exs @@ -0,0 +1,212 @@ +defmodule Coflux.EcsLauncherTest do + @moduledoc """ + What a failed ECS worker reports. + + ECS doesn't serve container output through its API, so the launcher + reads CloudWatch when it can and falls back to what ECS itself said + when it can't. Everything here is about that fallback holding: a worker + that died is recorded either way, and no failure to read logs is + allowed to become a failed poll. + """ + use ExUnit.Case, async: true + + @config %{ + region: "eu-west-1", + cluster: "coflux", + task_definition: "coflux-worker", + access_key_id: "AKIATEST", + secret_access_key: "test-secret" + } + + @data %{ + task_arn: "arn:aws:ecs:eu-west-1:123456789012:task/coflux/abc123", + cluster: "coflux", + container_name: "worker" + } + + describe "a worker whose container exited" do + test "reports the tail of its log stream when the task definition logs to CloudWatch" do + {:ok, false, error, logs} = poll(stub()) + + assert error == "exit_code:1" + assert logs == "Usage:\n coflux worker\nunknown flag: --all-modules" + end + + test "asks for the stream the awslogs driver would have written to" do + poll(stub()) + + assert_receive {:logs_request, body} + + assert body["logGroupName"] == "/coflux/worker" + assert body["logStreamName"] == "worker/worker/abc123" + assert body["startFromHead"] == false + end + + test "signs the log request for the region the logs are in, not the cluster's" do + poll(stub(log_options: %{"awslogs-region" => "us-east-1"})) + + assert_receive {:logs_url, url} + assert url == "https://logs.us-east-1.amazonaws.com" + end + end + + describe "falling back when the logs can't be read" do + test "the role is not allowed to read them" do + {:ok, false, error, logs} = + poll(stub(logs_response: {400, %{"__type" => "AccessDeniedException"}})) + + assert error == "exit_code:1" + assert logs =~ "Essential container in task exited" + assert logs =~ "exit code 1" + end + + test "CloudWatch is unreachable" do + {:ok, false, _error, logs} = poll(stub(logs_response: :unreachable)) + assert logs =~ "Essential container in task exited" + end + + test "the task definition has no log configuration" do + {:ok, false, _error, logs} = poll(stub(log_driver: nil)) + assert logs =~ "Essential container in task exited" + end + + test "it logs somewhere other than CloudWatch" do + {:ok, false, _error, logs} = poll(stub(log_driver: "awsfirelens")) + assert logs =~ "Essential container in task exited" + end + + test "there is no stream prefix to build a stream name from" do + {:ok, false, _error, logs} = poll(stub(log_options: %{"awslogs-stream-prefix" => nil})) + assert logs =~ "Essential container in task exited" + end + + test "the task definition can't be described" do + {:ok, false, _error, logs} = + poll(stub(definition_response: {400, %{"__type" => "AccessDeniedException"}})) + + assert logs =~ "Essential container in task exited" + end + + test "the events haven't been delivered yet" do + {:ok, false, _error, logs} = poll(stub(events: [])) + assert logs =~ "Essential container in task exited" + end + + test "the container's own reason is included, since that's where anything specific is" do + {:ok, false, error, logs} = + poll(stub(log_driver: nil, container_reason: "OutOfMemoryError: container killed")) + + assert error == "oom_killed" + assert logs =~ "OutOfMemoryError: container killed" + end + end + + describe "a worker that stopped for a reason that isn't a failure" do + test "reports no error and no logs, and never reads any" do + {:ok, false, error, logs} = poll(stub(stop_code: "UserInitiated")) + + assert error == nil + assert logs == nil + refute_receive {:logs_request, _} + end + end + + # --- Helpers --- + + defp poll(adapter) do + Coflux.EcsLauncher.poll(@data, Map.put(@config, :req_options, adapter: adapter)) + end + + # A stubbed ECS/CloudWatch pair. Routes on the x-amz-target header, so + # one adapter answers all three calls a failing poll makes. + defp stub(opts \\ []) do + test = self() + + stop_code = Keyword.get(opts, :stop_code, "EssentialContainerExited") + log_driver = Keyword.get(opts, :log_driver, "awslogs") + container_reason = Keyword.get(opts, :container_reason) + definition_response = Keyword.get(opts, :definition_response) + logs_response = Keyword.get(opts, :logs_response) + events = Keyword.get(opts, :events, default_events()) + + log_options = + %{"awslogs-group" => "/coflux/worker", "awslogs-stream-prefix" => "worker"} + |> Map.merge(Keyword.get(opts, :log_options, %{})) + |> Enum.reject(fn {_k, v} -> is_nil(v) end) + |> Map.new() + + fn request -> + target = request.headers |> Map.new() |> Map.get("x-amz-target") |> List.first() + body = Jason.decode!(request.body) + + case target do + "AmazonEC2ContainerServiceV20141113.DescribeTasks" -> + {request, json(200, describe_tasks(stop_code, container_reason))} + + "AmazonEC2ContainerServiceV20141113.DescribeTaskDefinition" -> + case definition_response do + {status, payload} -> {request, json(status, payload)} + nil -> {request, json(200, describe_task_definition(log_driver, log_options))} + end + + "Logs_20140328.GetLogEvents" -> + send(test, {:logs_request, body}) + send(test, {:logs_url, to_string(request.url)}) + + case logs_response do + {status, payload} -> {request, json(status, payload)} + :unreachable -> {request, %Req.TransportError{reason: :econnrefused}} + nil -> {request, json(200, %{"events" => events})} + end + end + end + end + + defp default_events do + [ + %{"message" => "Usage:"}, + %{"message" => " coflux worker"}, + %{"message" => "unknown flag: --all-modules"} + ] + end + + defp describe_tasks(stop_code, container_reason) do + container = + %{"name" => "worker", "exitCode" => 1} + |> then(&if container_reason, do: Map.put(&1, "reason", container_reason), else: &1) + + %{ + "tasks" => [ + %{ + "taskArn" => "arn:aws:ecs:eu-west-1:123456789012:task/coflux/abc123", + "taskDefinitionArn" => "arn:aws:ecs:eu-west-1:123456789012:task-definition/w:1", + "lastStatus" => "STOPPED", + "stopCode" => stop_code, + "stoppedReason" => "Essential container in task exited", + "containers" => [container] + } + ] + } + end + + defp describe_task_definition(nil, _options) do + %{"taskDefinition" => %{"containerDefinitions" => [%{"name" => "worker"}]}} + end + + defp describe_task_definition(driver, options) do + %{ + "taskDefinition" => %{ + "containerDefinitions" => [ + %{ + "name" => "worker", + "logConfiguration" => %{"logDriver" => driver, "options" => options} + } + ] + } + } + end + + defp json(status, payload) do + Req.Response.new(status: status, body: Jason.encode!(payload)) + end +end From 2abdc5aad37e10171da01f36919f6cc1c11a853e Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 21:12:19 +0100 Subject: [PATCH 18/21] Fix idle timeout units conversion --- .../coflux/orchestration/server/scheduler.ex | 6 ++--- tests/test_pools.py | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 2bac0f8c..79a89fb4 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -780,12 +780,12 @@ defmodule Coflux.Orchestration.Server.Scheduler do ) end - # A pool can say how long its workers linger once idle, in seconds: a - # worker that takes a while to start is worth keeping warm between + # A pool can say how long its workers linger once idle, in milliseconds: + # a worker that takes a while to start is worth keeping warm between # runs. One whose pool doesn't say gets the default. defp worker_idle_timeout_ms(state, worker) do case get_in(state.pools, [worker.workspace_id, worker.pool_name, :idle_timeout_ms]) do - seconds when is_integer(seconds) and seconds >= 0 -> seconds * 1000 + timeout_ms when is_integer(timeout_ms) and timeout_ms >= 0 -> timeout_ms _ -> @default_worker_idle_timeout_ms end end diff --git a/tests/test_pools.py b/tests/test_pools.py index 573251cd..66742833 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -364,6 +364,28 @@ def test_idle_timeout_keeps_worker_warm(self, pool_env): executor.wait_connections(1, timeout=1) assert len(cli.pools_launches("warm-pool", host=host)) == 1 + def test_idle_timeout_stops_worker(self, pool_env): + """A worker past its pool's idle timeout is stopped. The timeout is + milliseconds all the way through: a value misread as seconds would + keep this worker for the best part of an hour.""" + host = pool_env["host"] + executor = pool_env["executor"] + targets = [workflow("test", "greet", parameters=["name"])] + _setup_pool(pool_env, targets, pool_name="brief-pool", idle_timeout="2s") + + resp = cli.submit("test/greet", '"one"', host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="one") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + # The timeout plus the sweep that enforces it, with room to spare. + worker = _wait_for_worker( + host, "brief-pool", lambda w: w["stoppedAt"] is not None, timeout=20 + ) + assert worker["stopError"] is None + assert worker["error"] is None + def test_update_common_fields(self, pool_env): """Common launcher fields can be updated on an existing pool.""" host = pool_env["host"] From 445760250be17e8265587e678acadc016dd75481 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Sun, 20 Sep 2026 21:37:28 +0100 Subject: [PATCH 19/21] Send initial stop signal to pool workers through connection --- cli/CHANGELOG.md | 1 + cli/cmd/coflux/worker.go | 17 +- cli/internal/worker/worker.go | 28 +++- docs/docs/pools.md | 9 ++ server/CHANGELOG.md | 2 + server/lib/coflux/events.ex | 13 +- server/lib/coflux/handlers/worker.ex | 6 + server/lib/coflux/orchestration/server.ex | 1 + .../coflux/orchestration/server/scheduler.ex | 149 +++++++++++++----- .../coflux/orchestration/server/snapshots.ex | 6 +- .../lib/coflux/orchestration/server/state.ex | 8 +- server/lib/coflux/orchestration/workers.ex | 14 +- server/lib/coflux/topics/pool.ex | 23 ++- tests/support/ecs.py | 12 ++ tests/test_pools.py | 57 ++++++- 15 files changed, 275 insertions(+), 71 deletions(-) diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 12131144..529138ac 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -5,6 +5,7 @@ Enhancements: - Adds `--type ecs` support for `pools create` and `pools update`, including the `roleArn` and `roleExternalId` fields for a role to assume before calling ECS. - Adds the `idleTimeout` pool field, for how long a pool keeps an idle worker before stopping it (`--set idleTimeout=5m`, exported as `idle_timeout = "5m"`). - Adds `secrets set`, `secrets list` and `secrets delete`. Pools refer to secrets by name (`tokenSecret`, `credentialsSecret`, `envSecrets`) instead of holding credentials, so `pools export` no longer needs `--include-secrets`. Each secret is set for one or more workspace patterns, given as a required `--workspaces`, in the same language `tokens create --workspaces` uses. +- The worker exits when the server asks it to (a `stop` command over its connection, sent to a pool worker that has been idle past its pool's timeout), draining in-flight executions first as it does on SIGTERM. Changes: diff --git a/cli/cmd/coflux/worker.go b/cli/cmd/coflux/worker.go index 1a90612d..d5148c82 100644 --- a/cli/cmd/coflux/worker.go +++ b/cli/cmd/coflux/worker.go @@ -189,19 +189,23 @@ func runWorker(cmd *cobra.Command, args []string) error { workerDone <- w.Run(ctx, modules, shouldRegister) }() + // A stop the server asks for is handled the way a signal is: drain + // what's running, then leave. select { case <-shutdownCh: - drainWorker(w, workerDrainTimeout, drainAbortCh, logger) - cancel() - <-workerDone - logger.Info("worker stopped") - return nil + case <-w.StopRequested(): case err := <-workerDone: if err != nil { return fmt.Errorf("worker error: %w", err) } return nil } + + drainWorker(w, workerDrainTimeout, drainAbortCh, logger) + cancel() + <-workerDone + logger.Info("worker stopped") + return nil } // drainWorker runs a graceful drain with the configured timeout, aborting @@ -300,6 +304,9 @@ func runWorkerWithWatch( case <-shutdownCh: reason = "shutdown" + case <-w.StopRequested(): + reason = "shutdown" + case err := <-workerDone: runCancel() running = false diff --git a/cli/internal/worker/worker.go b/cli/internal/worker/worker.go index 6ea63268..52b53eb9 100644 --- a/cli/internal/worker/worker.go +++ b/cli/internal/worker/worker.go @@ -68,7 +68,13 @@ type Worker struct { connMu sync.RWMutex conn *api.Connection - connCh chan struct{} // closed when a new connection is established + + // Closed once the server has asked the worker to stop (a `stop` + // command over the connection), which the command treats the way it + // treats SIGTERM: drain, then exit. + stopCh chan struct{} + stopOnce sync.Once + connCh chan struct{} // closed when a new connection is established mu sync.RWMutex executions map[string]*executionState @@ -137,11 +143,19 @@ func New(cfg *config.Config, adp adapter.Adapter, session string, logger *slog.L session: session, logger: logger, connCh: make(chan struct{}), + stopCh: make(chan struct{}), executions: make(map[string]*executionState), streamSubs: make(map[streamSubKey]*streamSubscription), } } +// StopRequested returns a channel that is closed once the server has asked +// the worker to stop. The worker keeps running until whoever runs it +// drains and cancels it, so in-flight executions still finish. +func (w *Worker) StopRequested() <-chan struct{} { + return w.stopCh +} + // getConn returns the current connection (thread-safe) // Returns nil if not connected - caller must handle this case func (w *Worker) getConn() *api.Connection { @@ -430,6 +444,7 @@ func (w *Worker) runConnection(ctx context.Context, targets map[string]map[strin ) conn.RegisterHandler("execute", w.handleExecute) conn.RegisterHandler("abort", w.handleAbort) + conn.RegisterHandler("stop", w.handleStop) conn.RegisterHandler("stream_items", w.handleStreamItems) conn.RegisterHandler("stream_closed", w.handleStreamClosed) conn.RegisterHandler("stream_demand", w.handleStreamDemand) @@ -740,6 +755,17 @@ func (w *Worker) refsToAdapter(refs []api.Reference) ([][]any, error) { return result, nil } +// handleStop is the server asking the worker to exit: it has been idle +// past its pool's timeout, or its pool is going away. Signalled once; +// a repeat (after a reconnect, say) changes nothing. +func (w *Worker) handleStop(_ []any) error { + w.stopOnce.Do(func() { + w.logger.Info("stop requested by server") + close(w.stopCh) + }) + return nil +} + func (w *Worker) handleAbort(params []any) error { if len(params) < 1 { return fmt.Errorf("abort: insufficient params") diff --git a/docs/docs/pools.md b/docs/docs/pools.md index 9162490b..4e97f8f5 100644 --- a/docs/docs/pools.md +++ b/docs/docs/pools.md @@ -209,6 +209,15 @@ These fields apply to all launcher types: | `env` | Environment variables (e.g., `--set env.KEY=VALUE`) | | `envSecrets` | Environment variables set from secrets (e.g., `--set envSecrets.API_KEY=api-key`) | +When a worker is to be stopped - it has sat idle for `idleTimeout`, or +its pool has been disabled or deleted - the server asks it to exit over +its own connection, and the worker drains and leaves as it does on +SIGTERM. The launcher is only asked to stop it if the worker isn't +connected, or hasn't gone within 30 seconds; so a worker still leaves +cleanly when the launcher can't be reached, and the launcher still +bounds one that doesn't respond. A worker is reported as stopped once +its launcher confirms it has gone, not once it was asked. + ### Modules A pool is chosen for an execution by the execution's module. The pool's diff --git a/server/CHANGELOG.md b/server/CHANGELOG.md index 0d6422f1..cc803b3c 100644 --- a/server/CHANGELOG.md +++ b/server/CHANGELOG.md @@ -4,11 +4,13 @@ Enhancements: - A worker launched on ECS that fails now carries the tail of its log stream, read from CloudWatch, the way the Docker, Kubernetes and process launchers already carry the container's own output. It needs the task definition to use the `awslogs` driver and the launcher's identity to have `logs:GetLogEvents`; without either, the worker reports what ECS said about the stop — now including the container's own reason and exit code rather than only the task-level message, which for a container exit says nothing but that it exited. - The ECS launcher can assume an IAM role (`roleArn`, with `roleExternalId` where the trust policy asks for one) with whatever credentials it finds, so a server in one account can launch into another. The search of the server's surroundings now includes a web identity token (`AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN`, as EKS sets for a pod whose service account has a role). Credentials issued by STS are cached until shortly before they expire. +- A worker that is to be stopped is asked to exit over its own connection (a `stop` command, which a CLI of this version acts on), and its launcher is only asked to stop it if it isn't connected or hasn't gone within 30 seconds. So a worker leaves cleanly even when the launcher can't be reached — with expired credentials, say — and the launcher still bounds a worker that doesn't respond. Changes: - Durations carry their unit in their name, and are milliseconds everywhere the API and the database speak them. `delay` is `delayMs`, `timeout` is `timeoutMs`, `maxAge` is `maxAgeMs`, `backoffMin`/`backoffMax` are `backoffMinMs`/`backoffMaxMs` — on `submit_workflow`, on `register_manifests`, and in the topics that report them. The worker protocol matches, so `max_age`, `backoff_min` and `backoff_max` gain the `_ms` the adapter already sent them with. Database columns are renamed to match; no values change. - The pool `idleTimeout` field, added earlier in this release and never published, is `idleTimeoutMs` on the API and is milliseconds rather than seconds. Its default is unchanged at 5 seconds. (The CLI still writes it as `idleTimeout`, taking a duration.) +- In the pool topic, a worker's `stoppedAt` is set when its launcher confirms it has gone, having been asked to stop, rather than when the launcher accepted the request — a task that is still stopping is no longer shown as stopped. `stoppingAt` is when the worker was first asked (over its connection or through the launcher), and `stopError` is how the latest attempt went. - Workspace patterns mean one thing everywhere now. A pattern selects a workspace (`development`), the workspaces under it (`development/*`, at any depth, but not `development` itself), or all of them (`*`) — the rule tokens already used, now used for a secret's workspaces too, where a bare name previously selected everything beneath it as well. Patterns that name nothing are rejected. - Setting a secret takes access containing every pattern given, whole, rather than access to any one workspace the pattern reaches — so a token for `staging` can no longer set a secret reaching `staging/feature-1`. - Secrets are set for one or more workspace patterns (`workspaces` on `set_secret` and `delete_secret`, replacing `scope`), stored once per pattern. Where patterns overlap, the nearest wins: an exact workspace, then a longer prefix, then a shorter one, then `*`. diff --git a/server/lib/coflux/events.ex b/server/lib/coflux/events.ex index e9ba040d..2ec4618a 100644 --- a/server/lib/coflux/events.ex +++ b/server/lib/coflux/events.ex @@ -408,13 +408,20 @@ defmodule Coflux.Events.WorkerLaunchResult do end defmodule Coflux.Events.WorkerStopping do - @moduledoc "Row: `worker_stops`." + @moduledoc """ + Row: `worker_stops`. One attempt to stop the worker - over its own + connection, or through its launcher. A worker can have several. + """ defstruct [:workspace, :pool, :worker, :stopping_at] end defmodule Coflux.Events.WorkerStopResult do - @moduledoc "Row: `worker_stop_results`." - defstruct [:workspace, :pool, :worker, :stopped_at, :error] + @moduledoc """ + Row: `worker_stop_results`. How the attempt went: an error if the + launcher refused, nil if the request was made. Neither means the worker + has gone - `WorkerDeactivated` says that. + """ + defstruct [:workspace, :pool, :worker, :completed_at, :error] end defmodule Coflux.Events.WorkerDeactivated do diff --git a/server/lib/coflux/handlers/worker.ex b/server/lib/coflux/handlers/worker.ex index 600c5758..0148f9b0 100644 --- a/server/lib/coflux/handlers/worker.ex +++ b/server/lib/coflux/handlers/worker.ex @@ -929,6 +929,12 @@ defmodule Coflux.Handlers.Worker do {[{:close, 4000, "workspace_not_found"}], state} end + # The server asking the worker to exit. The worker drains and leaves; + # nothing comes back over this connection but its close. + def websocket_info(:stop_worker, state) do + {[command_message("stop", [])], state} + end + defp is_recognised_execution?(execution_id, state) do MapSet.member?(state.execution_ids, execution_id) end diff --git a/server/lib/coflux/orchestration/server.ex b/server/lib/coflux/orchestration/server.ex index 9cf63464..320e24d2 100644 --- a/server/lib/coflux/orchestration/server.ex +++ b/server/lib/coflux/orchestration/server.ex @@ -152,6 +152,7 @@ defmodule Coflux.Orchestration.Server do session_id: nil, stop_id: nil, stop_retry_at: nil, + stop_signalled_at: nil, last_poll_at: nil, polling: false, poll_failures: 0, diff --git a/server/lib/coflux/orchestration/server/scheduler.ex b/server/lib/coflux/orchestration/server/scheduler.ex index 79a89fb4..3710577d 100644 --- a/server/lib/coflux/orchestration/server/scheduler.ex +++ b/server/lib/coflux/orchestration/server/scheduler.ex @@ -88,6 +88,12 @@ defmodule Coflux.Orchestration.Server.Scheduler do # How long before asking the launcher again after a stop fails. @stop_retry_interval_ms 30_000 + # How long a worker told to stop over its connection is given to leave + # before its launcher is asked to stop it instead. Long enough for a + # clean exit and the poll that notices it; asking the launcher as well + # is harmless, just a stop the record didn't need. + @stop_grace_ms 30_000 + @default_activation_timeout_ms 600_000 @default_reconnection_timeout_ms 30_000 @@ -525,6 +531,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do session_id: session_id, stop_id: nil, stop_retry_at: nil, + stop_signalled_at: nil, last_poll_at: nil, polling: false, poll_failures: 0, @@ -646,49 +653,21 @@ defmodule Coflux.Orchestration.Server.Scheduler do ) end) + # A worker is asked to leave over its own connection first; its + # launcher is only asked to stop it if it isn't connected, or hasn't + # gone within the grace. So a worker leaves cleanly even when the + # launcher can't be reached - expired credentials, say - and the + # launcher still bounds one that won't listen. state = state.workers |> Enum.filter(fn {_worker_id, worker} -> stop_due?(state, worker, now) end) |> Enum.reduce(state, fn {worker_id, worker}, state -> - {:ok, worker_stop_id, stopping_at} = Workers.create_worker_stop(state.db, worker_id) - - state = - state - |> update_worker(worker_id, &%{&1 | stop_id: worker_stop_id, stop_retry_at: nil}) - |> Effects.emit(%WorkerStopping{ - workspace: State.workspace_external_id(state, worker.workspace_id), - pool: worker.pool_name, - worker: worker.external_id, - stopping_at: stopping_at - }) - - case worker_launcher(state, worker) do - {:ok, launcher} -> - Fleet.call_launcher(state, launcher, :stop, [worker.data, launcher], fn state, - result -> - case result do - {:ok, :ok} -> - {:ok, stopped_at} = - Workers.create_worker_stop_result(state.db, worker_stop_id, nil) - - Effects.emit(state, %WorkerStopResult{ - workspace: State.workspace_external_id(state, worker.workspace_id), - pool: worker.pool_name, - worker: worker.external_id, - stopped_at: stopped_at, - error: nil - }) - - {:ok, {:error, reason}} -> - record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) - - :error -> - record_stop_failure(state, worker_id, worker, worker_stop_id, "stop_crashed") - end - end) + case worker_connection(state, worker) do + {:ok, pid} when is_nil(worker.stop_signalled_at) -> + signal_stop(state, worker_id, worker, pid, now) - {:error, reason} -> - record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) + _ -> + launcher_stop(state, worker_id, worker) end end) @@ -997,9 +976,10 @@ defmodule Coflux.Orchestration.Server.Scheduler do cond do # Nothing to ask the launcher about until the launch has landed. is_nil(worker.data) -> false - # A stop is already in flight, or has already succeeded. + # A launcher stop is in flight, or has already succeeded. worker.stop_id -> false - # A previous stop failed; wait before asking again. + # The worker was signalled and is being given time to leave, or a + # launcher stop failed and is waiting to be retried. worker.stop_retry_at && now < worker.stop_retry_at -> false # The session has gone, so there is nothing left to drain. is_nil(worker.session_id) -> true @@ -1015,6 +995,93 @@ defmodule Coflux.Orchestration.Server.Scheduler do end end + # Asks the worker itself to stop. There is no answer to wait for: the + # worker drains and exits, and the launcher's poll is what says it has + # gone - which is why this doesn't set `stop_id`, and instead arms the + # retry that asks the launcher if the worker is still there after the + # grace. The attempt is recorded as made, not as the worker stopped. + defp signal_stop(state, worker_id, worker, pid, now) do + {:ok, worker_stop_id, stopping_at} = Workers.create_worker_stop(state.db, worker_id) + {:ok, completed_at} = Workers.create_worker_stop_result(state.db, worker_stop_id, nil) + + send(pid, :stop_worker) + + state + |> update_worker( + worker_id, + &%{&1 | stop_signalled_at: now, stop_retry_at: now + @stop_grace_ms} + ) + |> Effects.emit(%WorkerStopping{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + stopping_at: stopping_at + }) + |> Effects.emit(%WorkerStopResult{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + completed_at: completed_at, + error: nil + }) + end + + # Asks the launcher to stop the worker. A launcher that accepts the + # request has only accepted it: the worker is stopped when a poll finds + # it gone, and deactivates it. + defp launcher_stop(state, worker_id, worker) do + {:ok, worker_stop_id, stopping_at} = Workers.create_worker_stop(state.db, worker_id) + + state = + state + |> update_worker(worker_id, &%{&1 | stop_id: worker_stop_id, stop_retry_at: nil}) + |> Effects.emit(%WorkerStopping{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + stopping_at: stopping_at + }) + + case worker_launcher(state, worker) do + {:ok, launcher} -> + Fleet.call_launcher(state, launcher, :stop, [worker.data, launcher], fn state, result -> + case result do + {:ok, :ok} -> + {:ok, completed_at} = + Workers.create_worker_stop_result(state.db, worker_stop_id, nil) + + Effects.emit(state, %WorkerStopResult{ + workspace: State.workspace_external_id(state, worker.workspace_id), + pool: worker.pool_name, + worker: worker.external_id, + completed_at: completed_at, + error: nil + }) + + {:ok, {:error, reason}} -> + record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) + + :error -> + record_stop_failure(state, worker_id, worker, worker_stop_id, "stop_crashed") + end + end) + + {:error, reason} -> + record_stop_failure(state, worker_id, worker, worker_stop_id, to_error(reason)) + end + end + + # The live connection of a worker's session, if it has one. + defp worker_connection(state, worker) do + with {:ok, session} <- worker_session(state, worker), + ref when not is_nil(ref) <- session.connection, + {pid, _session_id} <- Map.get(state.connections, ref) do + {:ok, pid} + else + _ -> :error + end + end + # A stop that failed is recorded as one: the container may well still be # running, and reporting it as stopped both misleads whoever is watching # and means nothing ever tries again. Clearing `stop_id` is what allows @@ -1027,7 +1094,7 @@ defmodule Coflux.Orchestration.Server.Scheduler do workspace: State.workspace_external_id(state, worker.workspace_id), pool: worker.pool_name, worker: worker.external_id, - stopped_at: nil, + completed_at: nil, error: error }) |> update_worker( diff --git a/server/lib/coflux/orchestration/server/snapshots.ex b/server/lib/coflux/orchestration/server/snapshots.ex index c216ce68..e46129b0 100644 --- a/server/lib/coflux/orchestration/server/snapshots.ex +++ b/server/lib/coflux/orchestration/server/snapshots.ex @@ -537,7 +537,7 @@ defmodule Coflux.Orchestration.Server.Snapshots do workers = Enum.flat_map(pool_workers, fn {worker_id, worker_external_id, starting_at, started_at, - start_error, stopping_at, stopped_at, stop_error, + start_error, stopping_at, stop_completed_at, stop_error, deactivated_at, error, logs, total_executions} -> worker = Map.get(state.workers, worker_id) @@ -585,13 +585,13 @@ defmodule Coflux.Orchestration.Server.Snapshots do ], else: [] ) ++ - if(stopped_at || stop_error, + if(stop_completed_at || stop_error, do: [ %WorkerStopResult{ workspace: workspace_external_id, pool: pool_name, worker: worker_external_id, - stopped_at: stopped_at, + completed_at: stop_completed_at, error: stop_error } ], diff --git a/server/lib/coflux/orchestration/server/state.ex b/server/lib/coflux/orchestration/server/state.ex index 1ab2e163..e2cbba36 100644 --- a/server/lib/coflux/orchestration/server/state.ex +++ b/server/lib/coflux/orchestration/server/state.ex @@ -88,7 +88,7 @@ defmodule Coflux.Orchestration.Server.State do # named on the wire. workspace_names: %{}, - # worker_id -> %{created_at, pool_id, pool_name, workspace_id, state, data, session_id, stop_id, stop_retry_at, last_poll_at, polling, poll_failures, first_poll_failure_at} + # worker_id -> %{created_at, pool_id, pool_name, workspace_id, state, data, session_id, stop_id, stop_retry_at, stop_signalled_at, last_poll_at, polling, poll_failures, first_poll_failure_at} # # Workers this server launched and hasn't deactivated. # `session_id` links to the session that connected for it, if @@ -96,6 +96,12 @@ defmodule Coflux.Orchestration.Server.State do # any that were launched but never got a session - nothing is # going to connect to those. # + # `stop_signalled_at` is when the worker was told to stop over + # its connection; `stop_id` is a launcher stop that was + # accepted (or is in flight), and `stop_retry_at` is when to + # ask the launcher, whether because the signalled worker hasn't + # left or because the launcher refused last time. + # # `polling` is set while a poll is in flight, so a slow # launcher can't accumulate overlapping polls. The poll failure # counters track *transient* launcher errors only: a launcher diff --git a/server/lib/coflux/orchestration/workers.ex b/server/lib/coflux/orchestration/workers.ex index eafb4434..ef06423f 100644 --- a/server/lib/coflux/orchestration/workers.ex +++ b/server/lib/coflux/orchestration/workers.ex @@ -167,11 +167,17 @@ defmodule Coflux.Orchestration.Workers do # Pool names are unique per workspace, not per project, so the workspace # is part of identifying a pool - without it this returns the workers of # every same-named pool in the project. + # + # A worker can be asked to stop more than once: over its connection, + # then through its launcher, then again if the launcher refused. What's + # reported is when it was first asked, and how the latest attempt went. def get_pool_workers(db, workspace_id, pool_name, limit \\ 100) do case query( db, """ - SELECT w.id, w.external_id, w.created_at, r.created_at, r.error, s.created_at, sr.created_at, sr.error, + SELECT w.id, w.external_id, w.created_at, r.created_at, r.error, + (SELECT MIN(created_at) FROM worker_stops WHERE worker_id = w.id) AS stopping_at, + sr.created_at, sr.error, d.created_at, d.error, wl.content, (SELECT COUNT(*) FROM assignments AS a INNER JOIN sessions AS ses ON ses.id = a.session_id @@ -198,13 +204,13 @@ defmodule Coflux.Orchestration.Workers do {:ok, rows} -> {:ok, Enum.map(rows, fn {worker_id, external_id, created_at, started_at, start_error, - stopping_at, stopped_at, stop_error, deactivated_at, error, logs, - total_executions} -> + stopping_at, stop_completed_at, stop_error, deactivated_at, error, + logs, total_executions} -> # Launch and stop errors are stored encoded; deactivation errors # are not. Decode here so a worker read back from the database # carries the same shapes as one observed live. {worker_id, external_id, created_at, started_at, decode_error(start_error), - stopping_at, stopped_at, decode_error(stop_error), deactivated_at, error, logs, + stopping_at, stop_completed_at, decode_error(stop_error), deactivated_at, error, logs, total_executions} end)} end diff --git a/server/lib/coflux/topics/pool.ex b/server/lib/coflux/topics/pool.ex index 0cdb483d..ddf2bce8 100644 --- a/server/lib/coflux/topics/pool.ex +++ b/server/lib/coflux/topics/pool.ex @@ -171,19 +171,28 @@ defmodule Coflux.Topics.Pool.Model do def apply(model, %WorkerLaunchResult{} = e), do: update(model, e.worker, &%{&1 | started_at: e.started_at, start_error: e.error}) + # A worker may be asked to stop more than once (over its connection, + # then through its launcher): `stopping_at` is the first time it was + # asked, `stop_error` is how the latest attempt went, and `stopped_at` + # is set only once the worker is confirmed gone, cleanly, having been + # asked - a stop that was requested is not a worker that has stopped. def apply(model, %WorkerStopping{} = e), - do: update(model, e.worker, &%{&1 | stopping_at: e.stopping_at}) + do: update(model, e.worker, &%{&1 | stopping_at: &1.stopping_at || e.stopping_at}) def apply(model, %WorkerStopResult{} = e), - do: update(model, e.worker, &%{&1 | stopped_at: e.stopped_at, stop_error: e.error}) + do: update(model, e.worker, &%{&1 | stop_error: e.error}) def apply(model, %WorkerDeactivated{} = e), do: - update( - model, - e.worker, - &%{&1 | deactivated_at: e.deactivated_at, error: e.error, logs: e.logs} - ) + update(model, e.worker, fn worker -> + %{ + worker + | deactivated_at: e.deactivated_at, + error: e.error, + logs: e.logs, + stopped_at: if(worker.stopping_at && is_nil(e.error), do: e.deactivated_at) + } + end) def apply(model, %WorkerStateChanged{} = e), do: update(model, e.worker, &%{&1 | state: e.state}) diff --git a/tests/support/ecs.py b/tests/support/ecs.py index 00f93f05..a1a3f2c7 100644 --- a/tests/support/ecs.py +++ b/tests/support/ecs.py @@ -79,6 +79,18 @@ def task_arns(self): with self._lock: return list(self.tasks) + def pause(self, arn): + """Freeze a task's process, so it holds its connection open but acts + on nothing - the shape of a worker that won't respond.""" + with self._lock: + proc = self.tasks[arn]["proc"] + proc.send_signal(signal.SIGSTOP) + + def resume(self, arn): + with self._lock: + proc = self.tasks[arn]["proc"] + proc.send_signal(signal.SIGCONT) + def kill_with_oom(self, arn): """Kill a task's process the way an OOM kill looks from ECS.""" with self._lock: diff --git a/tests/test_pools.py b/tests/test_pools.py index 66742833..767cec1e 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -867,7 +867,9 @@ def test_export_names_secrets(self, pool_env, tmp_path): """An export carries the names, never the values, and imports back as long as the secrets exist.""" host = pool_env["host"] - cli.secrets_set("k8s-token", "super-secret-token", workspaces="default", host=host) + cli.secrets_set( + "k8s-token", "super-secret-token", workspaces="default", host=host + ) self._kubernetes_pool(host) cli._coflux( "pools", @@ -1026,8 +1028,42 @@ def test_runs_worker_as_task(self, ecs_env): assert "/us-east-1/ecs/aws4_request" in authorization assert "x-amz-date" in headers - def test_idle_worker_is_stopped(self, ecs_env): - """An idle worker's task is stopped, and a task stopped on request + def test_idle_worker_leaves_when_asked(self, ecs_env): + """An idle worker is asked to exit over its own connection and does, + so its task ends without ECS being asked to stop it - which is what + keeps an idle worker from running on when the launcher's + credentials have lapsed. It is recorded as stopped only once the + task is seen to have gone, and as a clean stop rather than a + failure.""" + host = ecs_env["host"] + executor = ecs_env["executor"] + fake = ecs_env["ecs"] + _setup_ecs_pool(ecs_env, [workflow("test", "greet")]) + + resp = cli.submit("test/greet", host=host) + executor.wait_connections(1, timeout=_LAUNCH_TIMEOUT) + ex = executor.next_execute(timeout=_EXEC_TIMEOUT) + ex.conn.complete(ex.execution_id, value="done") + poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["stoppingAt"] is not None + ) + # Asked, but not yet gone: not stopped. + if worker["deactivatedAt"] is None: + assert worker["stoppedAt"] is None + + worker = _wait_for_worker( + host, "ecs-pool", lambda w: w["deactivatedAt"] is not None + ) + assert worker["stoppedAt"] == worker["deactivatedAt"] + assert worker["stopError"] is None + assert worker["error"] is None + assert fake.requests_for("StopTask") == [] + + def test_unresponsive_worker_is_stopped_by_launcher(self, ecs_env): + """A worker that doesn't act on the stop it was sent is stopped + through ECS once it has had its grace, and a task stopped that way isn't reported as having failed.""" host = ecs_env["host"] executor = ecs_env["executor"] @@ -1040,13 +1076,22 @@ def test_idle_worker_is_stopped(self, ecs_env): ex.conn.complete(ex.execution_id, value="done") poll_result(resp["runId"], host, timeout=_RESULT_TIMEOUT) - [(stop_task, _)] = fake.wait_for("StopTask", timeout=30) - assert stop_task["cluster"] == fake.cluster - assert stop_task["task"] in fake.task_arns() + # Frozen, the worker keeps its connection but never reads the stop. + [arn] = fake.task_arns() + fake.pause(arn) + + # The idle timeout, then the grace the signalled worker is given. + [(stop_task, _)] = fake.wait_for("StopTask", timeout=60) + assert stop_task["task"] == arn + worker = _wait_for_worker(host, "ecs-pool", lambda w: w["stoppingAt"]) + assert worker["deactivatedAt"] is None + assert worker["stoppedAt"] is None + fake.resume(arn) worker = _wait_for_worker( host, "ecs-pool", lambda w: w["deactivatedAt"] is not None ) + assert worker["stoppedAt"] == worker["deactivatedAt"] assert worker["stopError"] is None assert worker["error"] is None From fd82d497fcee42bfe76b685e67d1ec09c1a77af2 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Tue, 22 Sep 2026 19:43:54 +0100 Subject: [PATCH 20/21] Format --- adapters/python/coflux/discovery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/adapters/python/coflux/discovery.py b/adapters/python/coflux/discovery.py index 483d6a68..6d8e52ea 100644 --- a/adapters/python/coflux/discovery.py +++ b/adapters/python/coflux/discovery.py @@ -21,7 +21,6 @@ serialize_streams, ) - # Top-level names that are never where targets live, and that tend to have # import-time side effects or dependencies the worker doesn't have. _SKIPPED_TOP_LEVEL = frozenset({"setup", "conftest", "tests", "test"}) From 16d71661e53d1cb7eee49028b75e9efd959f0297 Mon Sep 17 00:00:00 2001 From: Joe Freeman Date: Tue, 22 Sep 2026 19:46:28 +0100 Subject: [PATCH 21/21] Fix test --- tests/test_pools.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_pools.py b/tests/test_pools.py index 767cec1e..8c269765 100644 --- a/tests/test_pools.py +++ b/tests/test_pools.py @@ -1097,7 +1097,7 @@ def test_unresponsive_worker_is_stopped_by_launcher(self, ecs_env): def test_oom_killed_task_is_reported(self, ecs_env): """A task that ECS stops for exceeding its memory is reported as - such, with the task's stopped reason in place of a log tail.""" + such, with ECS's account of the stop in place of a log tail.""" host = ecs_env["host"] executor = ecs_env["executor"] fake = ecs_env["ecs"] @@ -1115,7 +1115,11 @@ def test_oom_killed_task_is_reported(self, ecs_env): host, "ecs-pool", lambda w: w["deactivatedAt"] is not None, timeout=45 ) assert worker["error"] == "oom_killed" - assert worker["logs"] == "Essential container in task exited" + assert worker["logs"] == ( + "Essential container in task exited" + " - OutOfMemoryError: Container killed due to memory usage" + " - exit code 137" + ) def test_refused_launch_is_reported(self, ecs_env): """A RunTask the API refuses fails the worker, with the API's own