From fce98f3241d6c53927e62d420892b5744ec64bf3 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 17:50:30 -0400 Subject: [PATCH 01/15] VMAgent: Add AWS access and secret key --- api/operator/v1beta1/vmagent_types.go | 4 + api/operator/v1beta1/vmagent_types_test.go | 47 +++++ api/operator/v1beta1/vmextra_types.go | 13 ++ docs/CHANGELOG.md | 1 + .../operator/factory/vmagent/vmagent.go | 37 +++- .../operator/factory/vmagent/vmagent_test.go | 161 +++++++++++++++++- 6 files changed, 258 insertions(+), 5 deletions(-) diff --git a/api/operator/v1beta1/vmagent_types.go b/api/operator/v1beta1/vmagent_types.go index c290ecb79e..7220959c52 100644 --- a/api/operator/v1beta1/vmagent_types.go +++ b/api/operator/v1beta1/vmagent_types.go @@ -372,6 +372,10 @@ type AWS struct { STSEndpoint string `json:"stsEndpoint,omitempty"` // UseSigv4 enables SigV4 request signing for the corresponding -remoteWrite.url UseSigv4 bool `json:"useSigv4,omitempty"` + // AccessKeySecret sets the AWS AccessKey for the -remoteWrite.url if useSigv4 is set. + AccessKeySecret *corev1.SecretKeySelector `json:"accessKeySecret,omitempty"` + // SecretKeySecret sets the SecretKey for the -remoteWrite.url if useSigv4 is set. + SecretKeySecret *corev1.SecretKeySelector `json:"secretKeySecret,omitempty"` } // VMAgentRemoteWriteSpec defines the remote storage configuration for VmAgent diff --git a/api/operator/v1beta1/vmagent_types_test.go b/api/operator/v1beta1/vmagent_types_test.go index 6812f8416c..bf45ae7c65 100644 --- a/api/operator/v1beta1/vmagent_types_test.go +++ b/api/operator/v1beta1/vmagent_types_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" ) @@ -182,3 +183,49 @@ func TestVMAgent_IsUnmanaged(t *testing.T) { Spec: VMAgentSpec{CommonScrapeParams: CommonScrapeParams{SelectAllByDefault: true}}, }, true) } + +func TestVMAgent_Validate_AWSRemoteWriteSettings(t *testing.T) { + f := func(rw VMAgentRemoteWriteSpec, wantErr bool) { + t.Helper() + rw.URL = "http://some-rw" + r := &VMAgent{Spec: VMAgentSpec{RemoteWrite: []VMAgentRemoteWriteSpec{rw}}} + if wantErr { + assert.Error(t, r.Validate()) + } else { + assert.NoError(t, r.Validate()) + } + } + // aws static credentials + validSel := &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}, + Key: "key", + } + f(VMAgentRemoteWriteSpec{AWS: &AWS{UseSigv4: true, AccessKeySecret: validSel, SecretKeySecret: validSel}}, false) + // must be set together + f(VMAgentRemoteWriteSpec{AWS: &AWS{UseSigv4: true, AccessKeySecret: validSel}}, true) + f(VMAgentRemoteWriteSpec{AWS: &AWS{UseSigv4: true, SecretKeySecret: validSel}}, true) + // silently ignored by vmagent without useSigv4 + f(VMAgentRemoteWriteSpec{AWS: &AWS{AccessKeySecret: validSel, SecretKeySecret: validSel}}, true) + // incomplete selector + f(VMAgentRemoteWriteSpec{AWS: &AWS{ + UseSigv4: true, + AccessKeySecret: &corev1.SecretKeySelector{Key: "key"}, + SecretKeySecret: validSel, + }}, true) + f(VMAgentRemoteWriteSpec{AWS: &AWS{ + UseSigv4: true, + AccessKeySecret: validSel, + SecretKeySecret: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}}, + }}, true) + // the pre-existing aws fields keep working on their own + f(VMAgentRemoteWriteSpec{AWS: &AWS{UseSigv4: true, Region: "us-east-1"}}, false) + + // extraEnvs must not collide with the operator-managed prefix + r := &VMAgent{Spec: VMAgentSpec{ + RemoteWrite: []VMAgentRemoteWriteSpec{{URL: "http://some-rw"}}, + CommonAppsParams: CommonAppsParams{ + ExtraEnvs: []corev1.EnvVar{{Name: RemoteWriteEnvPrefix + "0_AWS_ACCESS_KEY", Value: "x"}}, + }, + }} + assert.Error(t, r.Validate()) +} \ No newline at end of file diff --git a/api/operator/v1beta1/vmextra_types.go b/api/operator/v1beta1/vmextra_types.go index 6c6e3427ed..d5fe5e1655 100644 --- a/api/operator/v1beta1/vmextra_types.go +++ b/api/operator/v1beta1/vmextra_types.go @@ -100,6 +100,7 @@ const ( VMAuthLBServiceProxyTargetLabel = "operator.victoriametrics.com/vmauthlb-proxy-name" VMAuthLBServiceProxyJobNameLabel = "operator.victoriametrics.com/vmauthlb-proxy-job-name" KubeNodeEnvName = "KUBE_NODE_NAME" + RemoteWriteEnvPrefix = "VM_REMOTEWRITE_" // IgnoreConversionLabel this annotation disables updating of corresponding VMObject // must be added to annotation of VMObject // annotations: @@ -130,6 +131,18 @@ const ( ConditionDomainTypeAppliedSuffix = ".victoriametrics.com/Applied" ) +// RemoteWriteAWSAccessKeyEnvName returns the name of the environment variable +// which holds AWS accessKey for spec.remoteWrite with the given idx. +func RemoteWriteAWSAccessKeyEnvName(idx int) string { + return fmt.Sprintf("%s%d_AWS_ACCESS_KEY", RemoteWriteEnvPrefix, idx) +} + +// RemoteWriteAWSSecretKeyEnvName returns the name of the environment variable +// which holds AWS secretKey for spec.remoteWrite with the given idx. +func RemoteWriteAWSSecretKeyEnvName(idx int) string { + return fmt.Sprintf("%s%d_AWS_SECRET_KEY", RemoteWriteEnvPrefix, idx) +} + // skip validation, if object has annotation. func MustSkipCRValidation(cr client.Object) bool { return cr.GetAnnotations()[SkipValidationAnnotation] == SkipValidationValue diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5030c95e0f..f98c966ae4 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -22,6 +22,7 @@ aliases: * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VT apps to [v0.12.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.12.0). * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): allow `targetRef.crd.kind` to reference a `VTAgent`, so `vmauth` can route to it by name instead of a static URL. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687). +* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): add `spec.remoteWrite[].aws.accessKeySecret` and `spec.remoteWrite[].aws.secretKeySecret` fields for static AWS SigV4 credentials. The operator projects the referenced secrets into the `vmagent` container as environment variables and passes only their references to `-remoteWrite.aws.accessKey` and `-remoteWrite.aws.secretKey`, so the credentials never appear in the pod spec or in the process arguments. The `VMAgent` CRD must be updated to get this feature. See [#REPLACE_WITH_PR_NUMBER](https://github.com/VictoriaMetrics/operator/pull/REPLACE_WITH_PR_NUMBER). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when queries are provided by selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects. Previously such a settings-only base config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): fix the `victoria-metrics-alert` chart's legacy value aliases (`server.remote.read`, `server.remote.write`, `server.notifier.alertmanager`, nested `basicAuth`/`bearer` auth keys and a list-valued `server.notifier.url`) being silently dropped during conversion; they are now resolved the same way the chart does. Plaintext `remoteWrite` credentials are moved into a generated Secret like the other blocks. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). diff --git a/internal/controller/operator/factory/vmagent/vmagent.go b/internal/controller/operator/factory/vmagent/vmagent.go index 6e8e380952..fadba07abc 100644 --- a/internal/controller/operator/factory/vmagent/vmagent.go +++ b/internal/controller/operator/factory/vmagent/vmagent.go @@ -998,7 +998,7 @@ func sortMap(m map[string]string) []item { return kv } -func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]string, error) { +func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]string, []corev1.EnvVar, error) { maxDiskUsage := defaultMaxDiskUsage var queues int32 if cr.Spec.RemoteWriteSettings != nil { @@ -1011,6 +1011,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin } var args []string + var envs []corev1.EnvVar var hasAnyQueuesSet bool var hasAnyDiskUsagesSet bool var storageLimit int64 @@ -1053,6 +1054,8 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin awsService := build.NewFlag("-remoteWrite.aws.service", "") awsSTSEndpoint := build.NewFlag("-remoteWrite.aws.stsEndpoint", "") awsUseSigv4 := build.NewFlag("-remoteWrite.aws.useSigv4", "false") + awsAccessKey := build.NewFlag("-remoteWrite.aws.accessKey", "") + awsSecretKey := build.NewFlag("-remoteWrite.aws.secretKey", "") var streamAggrConfigs []*vmv1beta1.StreamAggrConfig var streamAggrKeys []string @@ -1168,6 +1171,32 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin awsService.Add(rw.AWS.Service, i) awsSTSEndpoint.Add(rw.AWS.STSEndpoint, i) awsUseSigv4.Add(strconv.FormatBool(rw.AWS.UseSigv4), i) + // AWS static credentials are projected into the container as env vars and + // only referenced from the flag value, so they never appear in the workload + // spec or in the process arguments. The secret is read here solely to fail + // the reconcile early instead of leaving the pod in CreateContainerConfigError. + if rw.AWS.AccessKeySecret != nil { + if _, err := ac.LoadKeyFromSecret(cr.Namespace, rw.AWS.AccessKeySecret); err != nil { + return nil, nil, fmt.Errorf("cannot load remoteWrite[%d].aws.accessKeySecret: %w", i, err) + } + envName := vmv1beta1.RemoteWriteAWSAccessKeyEnvName(i) + envs = append(envs, corev1.EnvVar{ + Name: envName, + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: rw.AWS.AccessKeySecret.DeepCopy()}, + }) + awsAccessKey.Add(fmt.Sprintf("%%{%s}", envName), i) + } + if rw.AWS.SecretKeySecret != nil { + if _, err := ac.LoadKeyFromSecret(cr.Namespace, rw.AWS.SecretKeySecret); err != nil { + return nil, nil, fmt.Errorf("cannot load remoteWrite[%d].aws.secretKeySecret: %w", i, err) + } + envName := vmv1beta1.RemoteWriteAWSSecretKeyEnvName(i) + envs = append(envs, corev1.EnvVar{ + Name: envName, + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: rw.AWS.SecretKeySecret.DeepCopy()}, + }) + awsSecretKey.Add(fmt.Sprintf("%%{%s}", envName), i) + } } streamAggrConfigs = append(streamAggrConfigs, rw.StreamAggrConfig) streamAggrKeys = append(streamAggrKeys, rw.AsConfigMapKey(i, "stream-aggr-conf")) @@ -1203,7 +1232,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin args = build.AppendFlagsToArgs(args, totalCount, oauth2ClientID, oauth2ClientSecretFile, oauth2Scopes, oauth2TokenURL) args = build.AppendFlagsToArgs(args, totalCount, headers, authPasswordFile, maxDiskUsagePerURL, forceVMProto) args = build.StreamAggrArgsTo(args, "remoteWrite.streamAggr", streamAggrKeys, streamAggrConfigs...) - args = build.AppendFlagsToArgs(args, totalCount, awsEC2Endpoint, awsRegion, awsRoleARN, awsService, awsSTSEndpoint, awsUseSigv4) + args = build.AppendFlagsToArgs(args, totalCount, awsEC2Endpoint, awsRegion, awsRoleARN, awsService, awsSTSEndpoint, awsUseSigv4, awsAccessKey, awsSecretKey) } if cr.Spec.RemoteWriteSettings != nil { @@ -1246,7 +1275,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin if !hasAnyDiskUsagesSet { args = append(args, fmt.Sprintf("-remoteWrite.maxDiskUsagePerURL=%s", maxDiskUsage)) } - return args, nil + return args, envs, nil } func deleteOrphaned(ctx context.Context, rclient client.Client, cr *vmv1beta1.VMAgent, baseConf *config.BaseOperatorConf) error { @@ -1371,7 +1400,7 @@ func createOrUpdateScrapeConfig(ctx context.Context, rclient client.Client, cr, } // ingest-only mode with remote write credentials: reconcile a credential-only secret // so vmagent can read credentials from files and the config-reloader can watch for changes. - if _, err := buildRemoteWriteArgs(cr, ac); err != nil { + if _, _, err := buildRemoteWriteArgs(cr, ac); err != nil { return 0, fmt.Errorf("cannot load remote write assets for ingest-only mode: %w", err) } owner := cr.AsOwner() diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index 5992695f98..702d10942f 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -963,7 +963,7 @@ func TestBuildRemoteWriteArgs(t *testing.T) { fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ac := getAssetsCache(ctx, fclient, o.cr) sort.Strings(o.want) - got, err := buildRemoteWriteArgs(o.cr, ac) + got, gotEnvs, err := buildRemoteWriteArgs(o.cr, ac) assert.NoError(t, err) sort.Strings(got) assert.Equal(t, o.want, got) @@ -3421,3 +3421,162 @@ func TestMakeSpecForAgentOk_WatchTargetDirPairing(t *testing.T) { } k8stools.AssertConfigReloaderWatchTargetDirs(t, got.Containers, want) } + +func TestBuildRemoteWriteArgs_PerURLFlags(t *testing.T) { + type opts struct { + remoteWrite []vmv1beta1.VMAgentRemoteWriteSpec + predefinedObjects []runtime.Object + // want lists only the flags under test, the always-present ones are added by the helper + want []string + wantEnvs []corev1.EnvVar + } + f := func(o opts) { + t.Helper() + ctx := context.Background() + cr := &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "default-vmagent", Namespace: "default"}, + Spec: vmv1beta1.VMAgentSpec{RemoteWrite: o.remoteWrite}, + } + urls := make([]string, 0, len(o.remoteWrite)) + for _, rw := range o.remoteWrite { + urls = append(urls, rw.URL) + } + want := append([]string{}, + "-remoteWrite.maxDiskUsagePerURL=1073741824", + "-remoteWrite.url="+strings.Join(urls, ","), + "-remoteWrite.tmpDataPath=/tmp/vmagent-remotewrite-data", + ) + want = append(want, o.want...) + sort.Strings(want) + + fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) + ac := getAssetsCache(ctx, fclient, cr) + got, gotEnvs, err := buildRemoteWriteArgs(cr, ac) + assert.NoError(t, err) + sort.Strings(got) + assert.Equal(t, want, got) + assert.Equal(t, o.wantEnvs, gotEnvs) + } + + // aws static credentials are passed by env var reference, never as a literal value + f(opts{ + remoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + {URL: "localhost:8429"}, + { + URL: "localhost:8430", + AWS: &vmv1beta1.AWS{ + UseSigv4: true, + Region: "us-east-1", + AccessKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "remote2-aws-secret"}, + Key: "accesskey", + }, + SecretKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "remote2-aws-secret"}, + Key: "secretkey", + }, + }, + }, + }, + predefinedObjects: []runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "remote2-aws-secret", Namespace: "default"}, + Data: map[string][]byte{ + "accesskey": []byte("accesskeytoken"), + "secretkey": []byte("secretkeytoken"), + }, + }, + }, + want: []string{ + `-remoteWrite.aws.region=,us-east-1`, + `-remoteWrite.aws.useSigv4=false,true`, + `-remoteWrite.aws.accessKey=,%{VM_REMOTEWRITE_1_AWS_ACCESS_KEY}`, + `-remoteWrite.aws.secretKey=,%{VM_REMOTEWRITE_1_AWS_SECRET_KEY}`, + }, + wantEnvs: []corev1.EnvVar{ + { + Name: "VM_REMOTEWRITE_1_AWS_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "remote2-aws-secret"}, + Key: "accesskey", + }}, + }, + { + Name: "VM_REMOTEWRITE_1_AWS_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "remote2-aws-secret"}, + Key: "secretkey", + }}, + }, + }, + }) +} + +func TestBuildRemoteWriteArgs_AWSSecretIsNotLeaked(t *testing.T) { + ctx := context.Background() + cr := &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{Name: "default-vmagent", Namespace: "default"}, + Spec: vmv1beta1.VMAgentSpec{ + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + { + URL: "localhost:8429", + AWS: &vmv1beta1.AWS{ + UseSigv4: true, + AccessKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}, + Key: "accesskey", + }, + SecretKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}, + Key: "secretkey", + }, + }, + }, + }, + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ExtraEnvs: []corev1.EnvVar{{Name: "SOME_ENV", Value: "some-value"}}, + }, + }, + } + fclient := k8stools.GetTestClientWithObjects([]runtime.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "aws-creds", Namespace: "default"}, + Data: map[string][]byte{ + "accesskey": []byte("SUPERSECRETACCESSKEY"), + "secretkey": []byte("SUPERSECRETSECRETKEY"), + }, + }, + }) + ac := getAssetsCache(ctx, fclient, cr) + podSpec, err := newPodSpec(cr, ac, 0) + assert.NoError(t, err) + + rendered, err := yaml.Marshal(podSpec) + assert.NoError(t, err) + assert.NotContains(t, string(rendered), "SUPERSECRETACCESSKEY", "aws accessKey must never reach the pod spec") + assert.NotContains(t, string(rendered), "SUPERSECRETSECRETKEY", "aws secretKey must never reach the pod spec") + + var envs []corev1.EnvVar + for _, c := range podSpec.Containers { + if c.Name == "vmagent" { + envs = c.Env + } + } + assert.Equal(t, []corev1.EnvVar{ + {Name: "SOME_ENV", Value: "some-value"}, + { + Name: "VM_REMOTEWRITE_0_AWS_ACCESS_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}, + Key: "accesskey", + }}, + }, + { + Name: "VM_REMOTEWRITE_0_AWS_SECRET_KEY", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-creds"}, + Key: "secretkey", + }}, + }, + }, envs, "operator-managed credentials must be appended after extraEnvs") +} \ No newline at end of file From 45ab354057c2d69e09f2124504e03bece54b597c Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:01:02 -0400 Subject: [PATCH 02/15] add AWS accessKeySecret and secretKeySecret --- docs/api.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api.md b/docs/api.md index cb3a36914d..f50a864107 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1827,6 +1827,8 @@ Appears in: [VMAgentRemoteWriteSpec (v1beta1)](#v1beta1-vmagentremotewritespec), | ec2Endpoint#
_string_ | _(Required)_
EC2Endpoint is an optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | region#
_string_ | _(Required)_
Region is an optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | roleARN#
_string_ | _(Required)_
RoleARN is an optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | +| accessKeySecret#
_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#secretkeyselector-v1-core)_ | _(Optional)_
AccessKeySecret sets the AWS AccessKey for the -remoteWrite.url if useSigv4 is set. | +| secretKeySecret#
_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#secretkeyselector-v1-core)_ | _(Optional)_
SecretKeySecret sets the SecretKey for the -remoteWrite.url if useSigv4 is set. | | service#
_string_ | _(Required)_
Service is an optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | stsEndpoint#
_string_ | _(Required)_
STSEndpoint is an optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | useSigv4#
_boolean_ | _(Required)_
UseSigv4 enables SigV4 request signing for the corresponding -remoteWrite.url | From f99661a1a3cfab32e29d914503bdbd8f0a4ded84 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:03:11 -0400 Subject: [PATCH 03/15] Add upgrade note --- docs/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f98c966ae4..bc00cfd97e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,6 +13,8 @@ aliases: ## tip +**Update note 1**: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): `VMAgent's` CRD must be updated. + * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VM apps to [v1.154.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.154.0) version * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default version for VMAnomaly to [v1.31.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1310). From 4592649b8a3b730c70497e85399b02f2b4b2a7db Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:03:54 -0400 Subject: [PATCH 04/15] Skip defining unused variable --- internal/controller/operator/factory/vmagent/vmagent_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/controller/operator/factory/vmagent/vmagent_test.go b/internal/controller/operator/factory/vmagent/vmagent_test.go index 702d10942f..fcd44cefc9 100644 --- a/internal/controller/operator/factory/vmagent/vmagent_test.go +++ b/internal/controller/operator/factory/vmagent/vmagent_test.go @@ -963,7 +963,7 @@ func TestBuildRemoteWriteArgs(t *testing.T) { fclient := k8stools.GetTestClientWithObjects(o.predefinedObjects) ac := getAssetsCache(ctx, fclient, o.cr) sort.Strings(o.want) - got, gotEnvs, err := buildRemoteWriteArgs(o.cr, ac) + got, _, err := buildRemoteWriteArgs(o.cr, ac) assert.NoError(t, err) sort.Strings(got) assert.Equal(t, o.want, got) @@ -3579,4 +3579,4 @@ func TestBuildRemoteWriteArgs_AWSSecretIsNotLeaked(t *testing.T) { }}, }, }, envs, "operator-managed credentials must be appended after extraEnvs") -} \ No newline at end of file +} From 84671d17e2d8757c014c0082c01807bcae2833d9 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:05:47 -0400 Subject: [PATCH 05/15] fix pass additional nil --- .../operator/factory/vmagent/vmagent.go | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/internal/controller/operator/factory/vmagent/vmagent.go b/internal/controller/operator/factory/vmagent/vmagent.go index fadba07abc..e1fb6f6200 100644 --- a/internal/controller/operator/factory/vmagent/vmagent.go +++ b/internal/controller/operator/factory/vmagent/vmagent.go @@ -532,11 +532,11 @@ func buildSTSServiceName(cr *vmv1beta1.VMAgent) string { func newPodSpec(cr *vmv1beta1.VMAgent, ac *build.AssetsCache, extraConfigSecretCount int) (*corev1.PodSpec, error) { var args []string - if rwArgs, err := buildRemoteWriteArgs(cr, ac); err != nil { + rwArgs, rwEnvs, err := buildRemoteWriteArgs(cr, ac) + if err != nil { return nil, fmt.Errorf("failed to build remote write args: %w", err) - } else { - args = append(args, rwArgs...) } + args = append(args, rwArgs...) cfg := config.MustGetBaseConfig() args = append(args, fmt.Sprintf("-httpListenAddr=:%s", cr.Spec.Port)) @@ -567,6 +567,8 @@ func newPodSpec(cr *vmv1beta1.VMAgent, ac *build.AssetsCache, extraConfigSecretC }, }) } + // operator-managed envs are appended after extraEnvs, Validate forbids extraEnvs with the same prefix + envs = append(envs, rwEnvs...) var ports []corev1.ContainerPort ports = append(ports, corev1.ContainerPort{Name: "http", Protocol: "TCP", ContainerPort: intstr.Parse(cr.Spec.Port).IntVal}) @@ -1079,7 +1081,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin if rw.TLSConfig != nil { creds, err := ac.BuildTLSCreds(cr.Namespace, rw.TLSConfig) if err != nil { - return nil, err + return nil, nil, err } tlsCAs.Add(creds.CAFile, i) tlsCerts.Add(creds.CertFile, i) @@ -1092,13 +1094,13 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin if rw.BasicAuth != nil { user, err := ac.LoadKeyFromSecret(cr.Namespace, &rw.BasicAuth.Username) if err != nil { - return nil, fmt.Errorf("cannot load BasicAuth username: %w", err) + return nil, nil, fmt.Errorf("cannot load BasicAuth username: %w", err) } authUser.Add(strconv.Quote(user), i) if len(rw.BasicAuth.Password.Name) > 0 { passFile, err := ac.LoadPathFromSecret(build.SecretConfigResourceKind, cr.Namespace, &rw.BasicAuth.Password) if err != nil { - return nil, fmt.Errorf("cannot load BasicAuth password: %w", err) + return nil, nil, fmt.Errorf("cannot load BasicAuth password: %w", err) } authPasswordFile.Add(passFile, i) } @@ -1109,7 +1111,7 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin if rw.BearerTokenSecret != nil && rw.BearerTokenSecret.Name != "" { value, err := ac.LoadPathFromSecret(build.SecretConfigResourceKind, cr.Namespace, rw.BearerTokenSecret) if err != nil { - return nil, fmt.Errorf("cannot load BearerTokenSecret: %w", err) + return nil, nil, fmt.Errorf("cannot load BearerTokenSecret: %w", err) } bearerTokenFile.Add(strconv.Quote(value), i) } @@ -1152,14 +1154,14 @@ func buildRemoteWriteArgs(cr *vmv1beta1.VMAgent, ac *build.AssetsCache) ([]strin if rw.OAuth2.ClientSecret != nil { oaSecretKeyFile, err := ac.LoadPathFromSecret(build.SecretConfigResourceKind, cr.Namespace, rw.OAuth2.ClientSecret) if err != nil { - return nil, fmt.Errorf("cannot load OAuth2 ClientSecret: %w", err) + return nil, nil, fmt.Errorf("cannot load OAuth2 ClientSecret: %w", err) } oauth2ClientSecretFile.Add(oaSecretKeyFile, i) } if len(rw.OAuth2.ClientID.PrefixedName()) > 0 { secret, err := ac.LoadKeyFromSecretOrConfigMap(cr.Namespace, &rw.OAuth2.ClientID) if err != nil { - return nil, err + return nil, nil, err } oauth2ClientID.Add(secret, i) } From 389af21906b0cc14c9f147ee874181060f6d0a38 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:06:19 -0400 Subject: [PATCH 06/15] update CRD with accessKeySecret and secretKeySecret --- config/crd/overlay/crd.descriptionless.yaml | 26 +++++++++++ config/crd/overlay/crd.yaml | 50 +++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/config/crd/overlay/crd.descriptionless.yaml b/config/crd/overlay/crd.descriptionless.yaml index d46703b8ad..b323714528 100644 --- a/config/crd/overlay/crd.descriptionless.yaml +++ b/config/crd/overlay/crd.descriptionless.yaml @@ -10996,12 +10996,38 @@ spec: properties: aws: properties: + accessKeySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic ec2Endpoint: type: string region: type: string roleARN: type: string + secretKeySecret: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic service: type: string stsEndpoint: diff --git a/config/crd/overlay/crd.yaml b/config/crd/overlay/crd.yaml index 71f0dffe4f..8ba9bccf8f 100644 --- a/config/crd/overlay/crd.yaml +++ b/config/crd/overlay/crd.yaml @@ -22184,6 +22184,31 @@ spec: aws: description: AWS describes params specific to AWS cloud properties: + accessKeySecret: + description: AccessKeySecret sets the AWS AccessKey for + the -remoteWrite.url if useSigv4 is set. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic ec2Endpoint: description: EC2Endpoint is an optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 @@ -22199,6 +22224,31 @@ spec: the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set type: string + secretKeySecret: + description: SecretKeySecret sets the SecretKey for the + -remoteWrite.url if useSigv4 is set. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic service: description: Service is an optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 From dea61f635b5fe215bf8c5efdd5c6a43daee71c2e Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:06:42 -0400 Subject: [PATCH 07/15] correct spacing --- api/operator/v1beta1/vmagent_types_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/operator/v1beta1/vmagent_types_test.go b/api/operator/v1beta1/vmagent_types_test.go index bf45ae7c65..d682be5dad 100644 --- a/api/operator/v1beta1/vmagent_types_test.go +++ b/api/operator/v1beta1/vmagent_types_test.go @@ -228,4 +228,4 @@ func TestVMAgent_Validate_AWSRemoteWriteSettings(t *testing.T) { }, }} assert.Error(t, r.Validate()) -} \ No newline at end of file +} From 9a94a09811813be62990a9d99d267b366098c0e3 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:07:25 -0400 Subject: [PATCH 08/15] add validation logic for secretKeySecret and accessKeySecret --- api/operator/v1beta1/vmagent_types.go | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/api/operator/v1beta1/vmagent_types.go b/api/operator/v1beta1/vmagent_types.go index 7220959c52..1431bf1201 100644 --- a/api/operator/v1beta1/vmagent_types.go +++ b/api/operator/v1beta1/vmagent_types.go @@ -176,10 +176,20 @@ func (cr *VMAgent) Validate() error { return err } } + for _, e := range cr.Spec.ExtraEnvs { + if strings.HasPrefix(e.Name, RemoteWriteEnvPrefix) { + return fmt.Errorf("spec.extraEnvs[%q] cannot use reserved prefix %q", e.Name, RemoteWriteEnvPrefix) + } + } for idx, rw := range cr.Spec.RemoteWrite { if rw.URL == "" { return fmt.Errorf("remoteWrite[%d].url cannot be empty", idx) } + if rw.AWS != nil { + if err := rw.AWS.validate(); err != nil { + return fmt.Errorf("bad remoteWrite[%d].aws: %w", idx, err) + } + } if len(rw.InlineUrlRelabelConfig) > 0 { if err := checkRelabelConfigs(rw.InlineUrlRelabelConfig); err != nil { return fmt.Errorf("bad remoteWrite[%d].urlRelabelingConfig: %w", idx, err) @@ -378,6 +388,24 @@ type AWS struct { SecretKeySecret *corev1.SecretKeySelector `json:"secretKeySecret,omitempty"` } +func (a *AWS) validate() error { + if (a.AccessKeySecret == nil) != (a.SecretKeySecret == nil) { + return fmt.Errorf("accessKeySecret and secretKeySecret must be set together") + } + if a.AccessKeySecret == nil { + return nil + } + if !a.UseSigv4 { + return fmt.Errorf("accessKeySecret and secretKeySecret require useSigv4=true") + } + for name, sel := range map[string]*corev1.SecretKeySelector{"accessKeySecret": a.AccessKeySecret, "secretKeySecret": a.SecretKeySecret} { + if sel.Name == "" || sel.Key == "" { + return fmt.Errorf("%s must have both name and key set", name) + } + } + return nil +} + // VMAgentRemoteWriteSpec defines the remote storage configuration for VmAgent // +k8s:openapi-gen=true type VMAgentRemoteWriteSpec struct { From 16ec12a68339fb297008a5fd01ed01b963b267fe Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:07:46 -0400 Subject: [PATCH 09/15] update spacing --- api/operator/v1beta1/vmextra_types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/operator/v1beta1/vmextra_types.go b/api/operator/v1beta1/vmextra_types.go index d5fe5e1655..125591ca21 100644 --- a/api/operator/v1beta1/vmextra_types.go +++ b/api/operator/v1beta1/vmextra_types.go @@ -100,7 +100,7 @@ const ( VMAuthLBServiceProxyTargetLabel = "operator.victoriametrics.com/vmauthlb-proxy-name" VMAuthLBServiceProxyJobNameLabel = "operator.victoriametrics.com/vmauthlb-proxy-job-name" KubeNodeEnvName = "KUBE_NODE_NAME" - RemoteWriteEnvPrefix = "VM_REMOTEWRITE_" + RemoteWriteEnvPrefix = "VM_REMOTEWRITE_" // IgnoreConversionLabel this annotation disables updating of corresponding VMObject // must be added to annotation of VMObject // annotations: From 485a1ad9bd20a616fa198ec6bc6d51e84d61deda Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 18:08:18 -0400 Subject: [PATCH 10/15] add accessKeySecret and secretKeySecret --- api/operator/v1alpha1/zz_generated.deepcopy.go | 2 +- api/operator/v1beta1/zz_generated.deepcopy.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/api/operator/v1alpha1/zz_generated.deepcopy.go b/api/operator/v1alpha1/zz_generated.deepcopy.go index 4d333456c8..6035c62cc1 100644 --- a/api/operator/v1alpha1/zz_generated.deepcopy.go +++ b/api/operator/v1alpha1/zz_generated.deepcopy.go @@ -706,7 +706,7 @@ func (in *VMDistributedZoneRemoteWriteSpec) DeepCopyInto(out *VMDistributedZoneR if in.AWS != nil { in, out := &in.AWS, &out.AWS *out = new(v1beta1.AWS) - **out = **in + (*in).DeepCopyInto(*out) } } diff --git a/api/operator/v1beta1/zz_generated.deepcopy.go b/api/operator/v1beta1/zz_generated.deepcopy.go index 11ac4a3bbe..f8cba20b85 100644 --- a/api/operator/v1beta1/zz_generated.deepcopy.go +++ b/api/operator/v1beta1/zz_generated.deepcopy.go @@ -67,6 +67,16 @@ func (in *APIServerConfig) DeepCopy() *APIServerConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AWS) DeepCopyInto(out *AWS) { *out = *in + if in.AccessKeySecret != nil { + in, out := &in.AccessKeySecret, &out.AccessKeySecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } + if in.SecretKeySecret != nil { + in, out := &in.SecretKeySecret, &out.SecretKeySecret + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWS. @@ -4865,7 +4875,7 @@ func (in *VMAgentRemoteWriteSpec) DeepCopyInto(out *VMAgentRemoteWriteSpec) { if in.AWS != nil { in, out := &in.AWS, &out.AWS *out = new(AWS) - **out = **in + (*in).DeepCopyInto(*out) } if in.Queues != nil { in, out := &in.Queues, &out.Queues From a87084f988e6bdf690be7e14f08bb2b0ecf468c8 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 22:01:56 -0400 Subject: [PATCH 11/15] Add E2EE test --- test/e2e/vmagent_test.go | 85 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/test/e2e/vmagent_test.go b/test/e2e/vmagent_test.go index 4ee665f78a..5bf0b1b863 100644 --- a/test/e2e/vmagent_test.go +++ b/test/e2e/vmagent_test.go @@ -29,6 +29,7 @@ var _ = Describe("test vmagent Controller", Label("vm", "agent", "vmagent"), fun Namespace: namespace, } tlsSecretName := "vmagent-remote-tls-certs" + awsSecretName := "vmagent-remote-aws-creds" AfterEach(func() { Expect(k8sClient.Delete(ctx, &vmv1beta1.VMAgent{ @@ -316,6 +317,90 @@ var _ = Describe("test vmagent Controller", Label("vm", "agent", "vmagent"), fun )).ToNot(HaveOccurred()) }), + Entry("with aws sigv4 static credentials", "remote-aws", + &vmv1beta1.VMAgent{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: nsn.Name, + }, + Spec: vmv1beta1.VMAgentSpec{ + CommonAppsParams: vmv1beta1.CommonAppsParams{ + ReplicaCount: ptr.To[int32](1), + }, + RemoteWrite: []vmv1beta1.VMAgentRemoteWriteSpec{ + { + URL: "http://localhost:8428", + AWS: &vmv1beta1.AWS{ + UseSigv4: true, + Region: "us-east-1", + AccessKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: awsSecretName, + }, + Key: "access-key", + }, + SecretKeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: awsSecretName, + }, + Key: "secret-key", + }, + }, + }, + }, + }, + }, + func() { + awsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: awsSecretName, + Namespace: namespace, + }, + StringData: map[string]string{ + "access-key": "E2EACCESSKEY", + "secret-key": "E2ESECRETKEY", + }, + } + Expect(func() error { + if err := k8sClient.Create(ctx, awsSecret); err != nil && + !k8serrors.IsAlreadyExists(err) { + return err + } + return nil + }()).ToNot(HaveOccurred()) + }, + func(cr *vmv1beta1.VMAgent) { + var dep appsv1.Deployment + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: cr.PrefixedName(), Namespace: namespace}, &dep)).ToNot(HaveOccurred()) + vmagentCnt := dep.Spec.Template.Spec.Containers[0] + Expect(vmagentCnt.Name).To(Equal("vmagent")) + Expect(vmagentCnt.Args).To(ContainElements( + "-remoteWrite.aws.accessKey=%{VM_REMOTEWRITE_0_AWS_ACCESS_KEY}", + "-remoteWrite.aws.secretKey=%{VM_REMOTEWRITE_0_AWS_SECRET_KEY}", + )) + Expect(vmagentCnt.Env).To(ContainElements( + corev1.EnvVar{Name: "VM_REMOTEWRITE_0_AWS_ACCESS_KEY", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: cr.Spec.RemoteWrite[0].AWS.AccessKeySecret}}, + corev1.EnvVar{Name: "VM_REMOTEWRITE_0_AWS_SECRET_KEY", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: cr.Spec.RemoteWrite[0].AWS.SecretKeySecret}}, + )) + // credentials must only be referenced, never inlined into the pod spec + Expect(fmt.Sprint(dep.Spec.Template.Spec)).ToNot(Or(ContainSubstring("E2EACCESSKEY"), ContainSubstring("E2ESECRETKEY"))) + Eventually(func() error { + return expectPodCount(ctx, k8sClient, &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Labels: cr.SelectorLabels(), + }, + }, 1) + }, eventualDeploymentPodTimeout, 1).ShouldNot(HaveOccurred()) + Expect(finalize.SafeDelete( + ctx, + k8sClient, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: awsSecretName, + Namespace: nsn.Namespace, + }}, + )).ToNot(HaveOccurred()) + }), Entry("with strict security", "strict-sec", &vmv1beta1.VMAgent{ ObjectMeta: metav1.ObjectMeta{ From 63db731b288b75c961a376c5ba08dc8e49037004 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Thu, 15 Oct 2026 22:39:22 -0400 Subject: [PATCH 12/15] Add PR number --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bc00cfd97e..884492e077 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,7 +24,7 @@ aliases: * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VT apps to [v0.12.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.12.0). * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): allow `targetRef.crd.kind` to reference a `VTAgent`, so `vmauth` can route to it by name instead of a static URL. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687). -* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): add `spec.remoteWrite[].aws.accessKeySecret` and `spec.remoteWrite[].aws.secretKeySecret` fields for static AWS SigV4 credentials. The operator projects the referenced secrets into the `vmagent` container as environment variables and passes only their references to `-remoteWrite.aws.accessKey` and `-remoteWrite.aws.secretKey`, so the credentials never appear in the pod spec or in the process arguments. The `VMAgent` CRD must be updated to get this feature. See [#REPLACE_WITH_PR_NUMBER](https://github.com/VictoriaMetrics/operator/pull/REPLACE_WITH_PR_NUMBER). +* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): add `spec.remoteWrite[].aws.accessKeySecret` and `spec.remoteWrite[].aws.secretKeySecret` fields for static AWS SigV4 credentials. The operator projects the referenced secrets into the `vmagent` container as environment variables and passes only their references to `-remoteWrite.aws.accessKey` and `-remoteWrite.aws.secretKey`, so the credentials never appear in the pod spec or in the process arguments. The `VMAgent` CRD must be updated to get this feature. See [#2737](https://github.com/VictoriaMetrics/operator/pull/2737). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when queries are provided by selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects. Previously such a settings-only base config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): fix the `victoria-metrics-alert` chart's legacy value aliases (`server.remote.read`, `server.remote.write`, `server.notifier.alertmanager`, nested `basicAuth`/`bearer` auth keys and a list-valued `server.notifier.url`) being silently dropped during conversion; they are now resolved the same way the chart does. Plaintext `remoteWrite` credentials are moved into a generated Secret like the other blocks. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). From edd8c3fde4d0bfd50e215948d0d2aa8ed10fd75f Mon Sep 17 00:00:00 2001 From: Stephan Burns <34520077+Sleuth56@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:31:05 -0400 Subject: [PATCH 13/15] Update docs/CHANGELOG.md Co-authored-by: Vadim Rutkovsky Signed-off-by: Stephan Burns <34520077+Sleuth56@users.noreply.github.com> --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 884492e077..9614bb8577 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -13,7 +13,7 @@ aliases: ## tip -**Update note 1**: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): `VMAgent's` CRD must be updated. +**Update note 1**: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): `VMAgent's` CRD must be updated after support for remote write AWS AccessKeySecret and SecretKeySecret was added. * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VM apps to [v1.154.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.154.0) version * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default version for VMAnomaly to [v1.31.0](https://docs.victoriametrics.com/anomaly-detection/changelog/#v1310). From c07ce3d3e43ad4602808b935ec19d533002f5d09 Mon Sep 17 00:00:00 2001 From: Stephan Burns <34520077+Sleuth56@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:39:53 -0400 Subject: [PATCH 14/15] Update docs/CHANGELOG.md Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com> Signed-off-by: Stephan Burns <34520077+Sleuth56@users.noreply.github.com> --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 9614bb8577..9232cb2624 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,7 +24,7 @@ aliases: * Dependency: [vmoperator](https://docs.victoriametrics.com/operator/): Updated default versions for VT apps to [v0.12.0](https://github.com/VictoriaMetrics/VictoriaTraces/releases/tag/v0.12.0). * FEATURE: [vmuser](https://docs.victoriametrics.com/operator/resources/vmuser/): allow `targetRef.crd.kind` to reference a `VTAgent`, so `vmauth` can route to it by name instead of a static URL. See [#2687](https://github.com/VictoriaMetrics/operator/issues/2687). -* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): add `spec.remoteWrite[].aws.accessKeySecret` and `spec.remoteWrite[].aws.secretKeySecret` fields for static AWS SigV4 credentials. The operator projects the referenced secrets into the `vmagent` container as environment variables and passes only their references to `-remoteWrite.aws.accessKey` and `-remoteWrite.aws.secretKey`, so the credentials never appear in the pod spec or in the process arguments. The `VMAgent` CRD must be updated to get this feature. See [#2737](https://github.com/VictoriaMetrics/operator/pull/2737). +* FEATURE: [vmagent](https://docs.victoriametrics.com/operator/resources/vmagent/): add `spec.remoteWrite[].aws.accessKeySecret` and `spec.remoteWrite[].aws.secretKeySecret` fields for static AWS SigV4 credentials. These fields keep credentials out of the pod spec and process arguments. The `VMAgent` CRD must be updated to get this feature. See [#2737](https://github.com/VictoriaMetrics/operator/pull/2737). * BUGFIX: [vmanomaly](https://docs.victoriametrics.com/operator/resources/vmanomaly/): allow `spec.configRawYaml` and `spec.configSecret` without `reader.queries` when queries are provided by selected [VMAnomalyConfig](https://docs.victoriametrics.com/operator/resources/vmanomalyconfig/) objects. Previously such a settings-only base config failed reconciliation with `reader.queries must be provided via configRawYaml or configSecret`. See [#2685](https://github.com/VictoriaMetrics/operator/pull/2685). * BUGFIX: [helm-converter](https://docs.victoriametrics.com/operator/helm-converter/): fix the `victoria-metrics-alert` chart's legacy value aliases (`server.remote.read`, `server.remote.write`, `server.notifier.alertmanager`, nested `basicAuth`/`bearer` auth keys and a list-valued `server.notifier.url`) being silently dropped during conversion; they are now resolved the same way the chart does. Plaintext `remoteWrite` credentials are moved into a generated Secret like the other blocks. See [#2601](https://github.com/VictoriaMetrics/operator/issues/2601). From 0e0c42a0650e765a12fe4d36297ad79bb0b8c9a5 Mon Sep 17 00:00:00 2001 From: Stephan Burns Date: Fri, 16 Oct 2026 17:00:14 -0400 Subject: [PATCH 15/15] Mark fields as optional --- api/operator/v1beta1/vmagent_types.go | 2 ++ docs/api.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/operator/v1beta1/vmagent_types.go b/api/operator/v1beta1/vmagent_types.go index 1431bf1201..65b9e9f960 100644 --- a/api/operator/v1beta1/vmagent_types.go +++ b/api/operator/v1beta1/vmagent_types.go @@ -383,8 +383,10 @@ type AWS struct { // UseSigv4 enables SigV4 request signing for the corresponding -remoteWrite.url UseSigv4 bool `json:"useSigv4,omitempty"` // AccessKeySecret sets the AWS AccessKey for the -remoteWrite.url if useSigv4 is set. + // +optional AccessKeySecret *corev1.SecretKeySelector `json:"accessKeySecret,omitempty"` // SecretKeySecret sets the SecretKey for the -remoteWrite.url if useSigv4 is set. + // +optional SecretKeySecret *corev1.SecretKeySelector `json:"secretKeySecret,omitempty"` } diff --git a/docs/api.md b/docs/api.md index f50a864107..4951a7ceee 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1824,10 +1824,10 @@ Appears in: [VMAgentRemoteWriteSpec (v1beta1)](#v1beta1-vmagentremotewritespec), | Field | Description | | --- | --- | +| accessKeySecret#
_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#secretkeyselector-v1-core)_ | _(Optional)_
AccessKeySecret sets the AWS AccessKey for the -remoteWrite.url if useSigv4 is set. | | ec2Endpoint#
_string_ | _(Required)_
EC2Endpoint is an optional AWS EC2 API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | region#
_string_ | _(Required)_
Region is an optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | roleARN#
_string_ | _(Required)_
RoleARN is an optional AWS region to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | -| accessKeySecret#
_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#secretkeyselector-v1-core)_ | _(Optional)_
AccessKeySecret sets the AWS AccessKey for the -remoteWrite.url if useSigv4 is set. | | secretKeySecret#
_[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.35/#secretkeyselector-v1-core)_ | _(Optional)_
SecretKeySecret sets the SecretKey for the -remoteWrite.url if useSigv4 is set. | | service#
_string_ | _(Required)_
Service is an optional AWS Service to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set | | stsEndpoint#
_string_ | _(Required)_
STSEndpoint is an optional AWS STS API endpoint to use for the corresponding -remoteWrite.url if -remoteWrite.aws.useSigv4 is set |