diff --git a/CHANGELOG.md b/CHANGELOG.md index d40a021f31..fade00616f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ENHANCEMENTS: * [The JSON representation of configuration](https://opentofu.org/docs/internals/json-format/#configuration-representation) returned by `tofu show` in `-json` mode now includes type constraint information for input variables and whether each input variable is required, in addition to the existing properties related to input variables. ([#3013](https://github.com/opentofu/opentofu/pull/3013)) * Multiline string updates in arrays are now diffed line-by-line, rather than as a single element, making it easier to see changes in the plan output. ([#3030](https://github.com/opentofu/opentofu/pull/3030)) * Add full support for -var, -var-file, and TF_VARS during `tofu apply` to support plan encryption ([#1998](https://github.com/opentofu/opentofu/pull/1998)) +* The S3 state backend now supports arguments to specify tags of the state and lock files. [#3038](https://github.com/opentofu/opentofu/pull/3038) BUG FIXES: diff --git a/internal/backend/remote-state/s3/backend.go b/internal/backend/remote-state/s3/backend.go index fb3e0a4ca9..2a16c44e5d 100644 --- a/internal/backend/remote-state/s3/backend.go +++ b/internal/backend/remote-state/s3/backend.go @@ -49,6 +49,8 @@ type Backend struct { serverSideEncryption bool customerEncryptionKey []byte acl string + lockTags map[string]string + stateTags map[string]string kmsKeyID string ddbTable string workspaceKeyPrefix string @@ -144,6 +146,16 @@ func (b *Backend) ConfigSchema() *configschema.Block { Optional: true, Description: "Canned ACL to be applied to the state file", }, + "state_tags": { + Type: cty.Map(cty.String), + Optional: true, + Description: "Tags to be applied to the state object", + }, + "lock_tags": { + Type: cty.Map(cty.String), + Optional: true, + Description: "Tags to be applied to the lock object", + }, "access_key": { Type: cty.String, Optional: true, @@ -673,6 +685,12 @@ func (b *Backend) Configure(ctx context.Context, obj cty.Value) tfdiags.Diagnost b.bucketName = stringAttr(obj, "bucket") b.keyName = stringAttr(obj, "key") b.acl = stringAttr(obj, "acl") + if val, ok := stringMapAttrOk(obj, "state_tags"); ok { + b.stateTags = val + } + if val, ok := stringMapAttrOk(obj, "lock_tags"); ok { + b.lockTags = val + } b.workspaceKeyPrefix = stringAttrDefault(obj, "workspace_key_prefix", "env:") b.serverSideEncryption = boolAttr(obj, "encrypt") b.kmsKeyID = stringAttr(obj, "kms_key_id") diff --git a/internal/backend/remote-state/s3/backend_state.go b/internal/backend/remote-state/s3/backend_state.go index bd1b2b3a1d..287b7dc7d9 100644 --- a/internal/backend/remote-state/s3/backend_state.go +++ b/internal/backend/remote-state/s3/backend_state.go @@ -140,6 +140,8 @@ func (b *Backend) remoteClient(name string) (*RemoteClient, error) { serverSideEncryption: b.serverSideEncryption, customerEncryptionKey: b.customerEncryptionKey, acl: b.acl, + stateTags: b.stateTags, + lockTags: b.lockTags, kmsKeyID: b.kmsKeyID, ddbTable: b.ddbTable, skipS3Checksum: b.skipS3Checksum, diff --git a/internal/backend/remote-state/s3/backend_test.go b/internal/backend/remote-state/s3/backend_test.go index d2a819f11a..d60ba055dc 100644 --- a/internal/backend/remote-state/s3/backend_test.go +++ b/internal/backend/remote-state/s3/backend_test.go @@ -1274,6 +1274,8 @@ func TestBackendExtraPaths(t *testing.T) { path: b.path("s1"), serverSideEncryption: b.serverSideEncryption, acl: b.acl, + stateTags: b.stateTags, + lockTags: b.lockTags, kmsKeyID: b.kmsKeyID, ddbTable: b.ddbTable, } diff --git a/internal/backend/remote-state/s3/client.go b/internal/backend/remote-state/s3/client.go index 371510abd9..48fba6fcbd 100644 --- a/internal/backend/remote-state/s3/client.go +++ b/internal/backend/remote-state/s3/client.go @@ -17,6 +17,7 @@ import ( "fmt" "io" "log" + "net/url" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -49,6 +50,8 @@ type RemoteClient struct { serverSideEncryption bool customerEncryptionKey []byte acl string + stateTags map[string]string + lockTags map[string]string kmsKeyID string ddbTable string @@ -207,6 +210,7 @@ func (c *RemoteClient) Put(ctx context.Context, data []byte) error { c.configurePutObjectChecksum(data, i) c.configurePutObjectEncryption(i) c.configurePutObjectACL(i) + c.configurePutObjectTags(i, c.stateTags) ctx, _ = attachLoggerToContext(ctx) @@ -322,6 +326,7 @@ func (c *RemoteClient) s3Lock(ctx context.Context, info *statemgr.LockInfo) erro c.configurePutObjectChecksum(lInfo, putParams) c.configurePutObjectEncryption(putParams) c.configurePutObjectACL(putParams) + c.configurePutObjectTags(putParams, c.lockTags) ctx, _ = attachLoggerToContext(ctx) @@ -648,6 +653,17 @@ func (c *RemoteClient) configurePutObjectACL(i *s3.PutObjectInput) { i.ACL = types.ObjectCannedACL(c.acl) } +func (c *RemoteClient) configurePutObjectTags(i *s3.PutObjectInput, tags map[string]string) { + if len(tags) == 0 { + return + } + headers := url.Values{} + for k, v := range tags { + headers.Add(k, v) + } + i.Tagging = aws.String(headers.Encode()) +} + const errBadChecksumFmt = `state data in S3 does not have the expected content. This may be caused by unusually long delays in S3 processing a previous state diff --git a/internal/backend/remote-state/s3/client_test.go b/internal/backend/remote-state/s3/client_test.go index 9e72570268..fe68134f24 100644 --- a/internal/backend/remote-state/s3/client_test.go +++ b/internal/backend/remote-state/s3/client_test.go @@ -980,6 +980,136 @@ func TestS3LockingWritingHeaders(t *testing.T) { } } +func TestS3LockObjectTagging(t *testing.T) { + // Configured the aws config the same way it is done for the backend to ensure a similar setup as the actual main logic. + _, awsCfg, _ := awsbase.GetAwsConfig(context.Background(), &awsbase.Config{Region: "us-east-1", AccessKey: "test", SecretKey: "key"}) + httpCl := &mockHttpClient{resp: &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(""))}} + s3Cl := s3.NewFromConfig(awsCfg, func(options *s3.Options) { + options.HTTPClient = httpCl + }) + + tests := []struct { + name string + action func(cl *RemoteClient) error + skipChecksum bool + + lockTags map[string]string + expectedTags string + }{ + { + name: "s3.Put with no tags", + skipChecksum: false, + lockTags: map[string]string{}, + expectedTags: "", + }, + { + name: "s3.Put with 1 tag", + skipChecksum: false, + lockTags: map[string]string{ + "a": "b", + }, + expectedTags: "a=b", + }, + { + name: "s3.Put with several tags", + skipChecksum: false, + lockTags: map[string]string{ + "a": "b", + "c": "e", + "f": "g", + }, + expectedTags: "a=b&c=e&f=g", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rc := RemoteClient{ + s3Client: s3Cl, + bucketName: "test-bucket", + path: "state-file", + useLockfile: true, + lockTags: tt.lockTags, + } + + _, err := rc.Lock(t.Context(), statemgr.NewLockInfo()) + if err != nil { + t.Fatalf("expected to have no error but got one: %s", err) + } + if httpCl.receivedReq == nil { + t.Fatal("request didn't reach the mock http client") + } + gotTags := httpCl.receivedReq.Header.Get("x-amz-tagging") + if gotTags != tt.expectedTags { + t.Errorf("Found tags %s did not match expected tags: %s", gotTags, tt.expectedTags) + } + }) + } +} + +func TestS3StateObjectTagging(t *testing.T) { + // Configured the aws config the same way it is done for the backend to ensure a similar setup as the actual main logic. + _, awsCfg, _ := awsbase.GetAwsConfig(context.Background(), &awsbase.Config{Region: "us-east-1", AccessKey: "test", SecretKey: "key"}) + httpCl := &mockHttpClient{resp: &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(""))}} + s3Cl := s3.NewFromConfig(awsCfg, func(options *s3.Options) { + options.HTTPClient = httpCl + }) + + tests := []struct { + name string + action func(cl *RemoteClient) error + skipChecksum bool + + stateTags map[string]string + expectedTags string + }{ + { + name: "s3.Put with no tags", + skipChecksum: false, + stateTags: map[string]string{}, + expectedTags: "", + }, + { + name: "s3.Put with 1 tag", + skipChecksum: false, + stateTags: map[string]string{ + "a": "b", + }, + expectedTags: "a=b", + }, + { + name: "s3.Put with several tags", + skipChecksum: false, + stateTags: map[string]string{ + "a": "b", + "c": "e", + "f": "g", + }, + expectedTags: "a=b&c=e&f=g", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rc := RemoteClient{ + s3Client: s3Cl, + bucketName: "test-bucket", + path: "state-file", + stateTags: tt.stateTags, + } + err := rc.Put(t.Context(), []byte("test")) + if err != nil { + t.Fatalf("expected to have no error but got one: %s", err) + } + if httpCl.receivedReq == nil { + t.Fatal("request didn't reach the mock http client") + } + gotTags := httpCl.receivedReq.Header.Get("x-amz-tagging") + if gotTags != tt.expectedTags { + t.Errorf("Found tags %s did not match expected tags: %s", gotTags, tt.expectedTags) + } + }) + } +} + // mockHttpClient is used to test the interaction of the s3 backend with the aws-sdk. // This is meant to be configured with a response that will be returned to the aws-sdk. // The receivedReq is going to contain the last request received by it. @@ -1011,3 +1141,53 @@ const encryptionPolicy = `{ } ] }` + +func TestConfigurePutObjectTags(t *testing.T) { + tests := []struct { + name string + tags map[string]string + result s3.PutObjectInput + }{ + { + name: "No tags", + tags: nil, + result: s3.PutObjectInput{}, + }, + { + name: "Basic tags", + tags: map[string]string{ + "abd": "def", + }, + result: s3.PutObjectInput{ + Tagging: aws.String("abd=def"), + }, + }, + { + name: "Complex values", + tags: map[string]string{ + "opentofu.com/environment": "production", + "my-special-chars": "&*&@$!(&)", + }, + result: s3.PutObjectInput{ + Tagging: aws.String("my-special-chars=%26%2A%26%40%24%21%28%26%29&opentofu.com%2Fenvironment=production"), + }, + }, + } + for _, tt := range tests { + c := RemoteClient{ + stateTags: tt.tags, + } + t.Run(tt.name, func(t *testing.T) { + pot := &s3.PutObjectInput{} + c.configurePutObjectTags(pot, c.stateTags) + + if pot.Tagging == nil && tt.result.Tagging == nil { + return + } + + if *pot.Tagging != *tt.result.Tagging { + t.Errorf("configurePutObjectTags() = %v; want %v", *pot.Tagging, *tt.result.Tagging) + } + }) + } +} diff --git a/website/docs/language/settings/backends/s3.mdx b/website/docs/language/settings/backends/s3.mdx index 8f6be75b2c..92aa9e9535 100644 --- a/website/docs/language/settings/backends/s3.mdx +++ b/website/docs/language/settings/backends/s3.mdx @@ -51,6 +51,11 @@ the target backend bucket: * `s3:PutObject` on `arn:aws:s3:::mybucket/path/to/my/key` * `s3:DeleteObject` on `arn:aws:s3:::mybucket/path/to/my/key` +OpenTofu may also need the following AWS IAM permissions on +the target backend bucket: + +* `s3:PutObjectTagging` on `arn:aws:s3:::mybucket/path/to/my/key` + This is seen in the following AWS IAM Statement: ```json @@ -342,6 +347,8 @@ The following configuration is required: The following configuration is optional: * `acl` - (Optional) [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) to be applied to the state file. +* `state_tags` - (Optional) [Tags](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) to be applied to the state object. +* `lock_tags` - (Optional) [Tags](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) to be applied to the lock object. * `encrypt` - (Optional) Enable [server side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html) of the state file. * `endpoint` - (Optional) **Deprecated** Custom endpoint for the AWS S3 API. This can also be sourced from the `AWS_S3_ENDPOINT` environment variable. * `force_path_style` - (Optional) **Deprecated** Enable path-style S3 URLs (`https:///` instead of `https://.`). Use `use_path_style` instead.