jprdonnelly/opentf
1
0
Fork 0
mirror of https://github.com/opentffoundation/opentf.git synced 2025-12-19 17:59:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add tag options for S3 state backend objects (#3038)

Browse Source 
Signed-off-by: Aaron George <aarongeorge1994+github@gmail.com>
Co-authored-by: Andrei Ciobanu <andreic9203@gmail.com>
This commit is contained in:
Aaron George
2025-08-07 08:39:19 +01:00
committed by GitHub
parent c5fd93482a
commit e802c63f58
7 changed files with 226 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -14,6 +14,7 @@ ENHANCEMENTS:
* [The JSON representation of configuration](https://opentofu.org/docs/internals/json-format/#configuration-representation) returned by `tofu show` in `-json` mode now includes type constraint information for input variables and whether each input variable is required, in addition to the existing properties related to input variables. ([#3013](https://github.com/opentofu/opentofu/pull/3013))
* Multiline string updates in arrays are now diffed line-by-line, rather than as a single element, making it easier to see changes in the plan output. ([#3030](https://github.com/opentofu/opentofu/pull/3030))
* Add full support for -var, -var-file, and TF_VARS during `tofu apply` to support plan encryption ([#1998](https://github.com/opentofu/opentofu/pull/1998))
* The S3 state backend now supports arguments to specify tags of the state and lock files. [#3038](https://github.com/opentofu/opentofu/pull/3038)
BUG FIXES:

18
internal/backend/remote-state/s3/backend.go
View File

@@ -49,6 +49,8 @@ type Backend struct {
serverSideEncryption bool
customerEncryptionKey []byte
acl string
lockTags map[string]string
stateTags map[string]string
kmsKeyID string
ddbTable string
workspaceKeyPrefix string
@@ -144,6 +146,16 @@ func (b *Backend) ConfigSchema() *configschema.Block {
Optional: true,
Description: "Canned ACL to be applied to the state file",
},
"state_tags": {
Type: cty.Map(cty.String),
Optional: true,
Description: "Tags to be applied to the state object",
},
"lock_tags": {
Type: cty.Map(cty.String),
Optional: true,
Description: "Tags to be applied to the lock object",
},
"access_key": {
Type: cty.String,
Optional: true,
@@ -673,6 +685,12 @@ func (b *Backend) Configure(ctx context.Context, obj cty.Value) tfdiags.Diagnost
b.bucketName = stringAttr(obj, "bucket")
b.keyName = stringAttr(obj, "key")
b.acl = stringAttr(obj, "acl")
if val, ok := stringMapAttrOk(obj, "state_tags"); ok {
b.stateTags = val
}
if val, ok := stringMapAttrOk(obj, "lock_tags"); ok {
b.lockTags = val
}
b.workspaceKeyPrefix = stringAttrDefault(obj, "workspace_key_prefix", "env:")
b.serverSideEncryption = boolAttr(obj, "encrypt")
b.kmsKeyID = stringAttr(obj, "kms_key_id")

2
internal/backend/remote-state/s3/backend_state.go
View File

@@ -140,6 +140,8 @@ func (b *Backend) remoteClient(name string) (*RemoteClient, error) {
serverSideEncryption: b.serverSideEncryption,
customerEncryptionKey: b.customerEncryptionKey,
acl: b.acl,
stateTags: b.stateTags,
lockTags: b.lockTags,
kmsKeyID: b.kmsKeyID,
ddbTable: b.ddbTable,
skipS3Checksum: b.skipS3Checksum,

2
internal/backend/remote-state/s3/backend_test.go
View File

@@ -1274,6 +1274,8 @@ func TestBackendExtraPaths(t *testing.T) {
path: b.path("s1"),
serverSideEncryption: b.serverSideEncryption,
acl: b.acl,
stateTags: b.stateTags,
lockTags: b.lockTags,
kmsKeyID: b.kmsKeyID,
ddbTable: b.ddbTable,
}

16
internal/backend/remote-state/s3/client.go
View File

@@ -17,6 +17,7 @@ import (
"fmt"
"io"
"log"
"net/url"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -49,6 +50,8 @@ type RemoteClient struct {
serverSideEncryption bool
customerEncryptionKey []byte
acl string
stateTags map[string]string
lockTags map[string]string
kmsKeyID string
ddbTable string
@@ -207,6 +210,7 @@ func (c *RemoteClient) Put(ctx context.Context, data []byte) error {
c.configurePutObjectChecksum(data, i)
c.configurePutObjectEncryption(i)
c.configurePutObjectACL(i)
c.configurePutObjectTags(i, c.stateTags)
ctx, _ = attachLoggerToContext(ctx)
@@ -322,6 +326,7 @@ func (c *RemoteClient) s3Lock(ctx context.Context, info *statemgr.LockInfo) erro
c.configurePutObjectChecksum(lInfo, putParams)
c.configurePutObjectEncryption(putParams)
c.configurePutObjectACL(putParams)
c.configurePutObjectTags(putParams, c.lockTags)
ctx, _ = attachLoggerToContext(ctx)
@@ -648,6 +653,17 @@ func (c *RemoteClient) configurePutObjectACL(i *s3.PutObjectInput) {
i.ACL = types.ObjectCannedACL(c.acl)
}
func (c *RemoteClient) configurePutObjectTags(i *s3.PutObjectInput, tags map[string]string) {
if len(tags) == 0 {
return
}
headers := url.Values{}
for k, v := range tags {
headers.Add(k, v)
}
i.Tagging = aws.String(headers.Encode())
}
const errBadChecksumFmt = `state data in S3 does not have the expected content.
This may be caused by unusually long delays in S3 processing a previous state

180
internal/backend/remote-state/s3/client_test.go
View File

@@ -980,6 +980,136 @@ func TestS3LockingWritingHeaders(t *testing.T) {
}
}
func TestS3LockObjectTagging(t *testing.T) {
// Configured the aws config the same way it is done for the backend to ensure a similar setup as the actual main logic.
_, awsCfg, _ := awsbase.GetAwsConfig(context.Background(), &awsbase.Config{Region: "us-east-1", AccessKey: "test", SecretKey: "key"})
httpCl := &mockHttpClient{resp: &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(""))}}
s3Cl := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
options.HTTPClient = httpCl
})
tests := []struct {
name string
action func(cl *RemoteClient) error
skipChecksum bool
lockTags map[string]string
expectedTags string
}{
{
name: "s3.Put with no tags",
skipChecksum: false,
lockTags: map[string]string{},
expectedTags: "",
},
{
name: "s3.Put with 1 tag",
skipChecksum: false,
lockTags: map[string]string{
"a": "b",
},
expectedTags: "a=b",
},
{
name: "s3.Put with several tags",
skipChecksum: false,
lockTags: map[string]string{
"a": "b",
"c": "e",
"f": "g",
},
expectedTags: "a=b&c=e&f=g",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rc := RemoteClient{
s3Client: s3Cl,
bucketName: "test-bucket",
path: "state-file",
useLockfile: true,
lockTags: tt.lockTags,
}
_, err := rc.Lock(t.Context(), statemgr.NewLockInfo())
if err != nil {
t.Fatalf("expected to have no error but got one: %s", err)
}
if httpCl.receivedReq == nil {
t.Fatal("request didn't reach the mock http client")
}
gotTags := httpCl.receivedReq.Header.Get("x-amz-tagging")
if gotTags != tt.expectedTags {
t.Errorf("Found tags %s did not match expected tags: %s", gotTags, tt.expectedTags)
}
})
}
}
func TestS3StateObjectTagging(t *testing.T) {
// Configured the aws config the same way it is done for the backend to ensure a similar setup as the actual main logic.
_, awsCfg, _ := awsbase.GetAwsConfig(context.Background(), &awsbase.Config{Region: "us-east-1", AccessKey: "test", SecretKey: "key"})
httpCl := &mockHttpClient{resp: &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(""))}}
s3Cl := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
options.HTTPClient = httpCl
})
tests := []struct {
name string
action func(cl *RemoteClient) error
skipChecksum bool
stateTags map[string]string
expectedTags string
}{
{
name: "s3.Put with no tags",
skipChecksum: false,
stateTags: map[string]string{},
expectedTags: "",
},
{
name: "s3.Put with 1 tag",
skipChecksum: false,
stateTags: map[string]string{
"a": "b",
},
expectedTags: "a=b",
},
{
name: "s3.Put with several tags",
skipChecksum: false,
stateTags: map[string]string{
"a": "b",
"c": "e",
"f": "g",
},
expectedTags: "a=b&c=e&f=g",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rc := RemoteClient{
s3Client: s3Cl,
bucketName: "test-bucket",
path: "state-file",
stateTags: tt.stateTags,
}
err := rc.Put(t.Context(), []byte("test"))
if err != nil {
t.Fatalf("expected to have no error but got one: %s", err)
}
if httpCl.receivedReq == nil {
t.Fatal("request didn't reach the mock http client")
}
gotTags := httpCl.receivedReq.Header.Get("x-amz-tagging")
if gotTags != tt.expectedTags {
t.Errorf("Found tags %s did not match expected tags: %s", gotTags, tt.expectedTags)
}
})
}
}
// mockHttpClient is used to test the interaction of the s3 backend with the aws-sdk.
// This is meant to be configured with a response that will be returned to the aws-sdk.
// The receivedReq is going to contain the last request received by it.
@@ -1011,3 +1141,53 @@ const encryptionPolicy = `{
}
]
}`
func TestConfigurePutObjectTags(t *testing.T) {
tests := []struct {
name string
tags map[string]string
result s3.PutObjectInput
}{
{
name: "No tags",
tags: nil,
result: s3.PutObjectInput{},
},
{
name: "Basic tags",
tags: map[string]string{
"abd": "def",
},
result: s3.PutObjectInput{
Tagging: aws.String("abd=def"),
},
},
{
name: "Complex values",
tags: map[string]string{
"opentofu.com/environment": "production",
"my-special-chars": "&*&@$!(&)",
},
result: s3.PutObjectInput{
Tagging: aws.String("my-special-chars=%26%2A%26%40%24%21%28%26%29&opentofu.com%2Fenvironment=production"),
},
},
}
for _, tt := range tests {
c := RemoteClient{
stateTags: tt.tags,
}
t.Run(tt.name, func(t *testing.T) {
pot := &s3.PutObjectInput{}
c.configurePutObjectTags(pot, c.stateTags)
if pot.Tagging == nil && tt.result.Tagging == nil {
return
}
if *pot.Tagging != *tt.result.Tagging {
t.Errorf("configurePutObjectTags() = %v; want %v", *pot.Tagging, *tt.result.Tagging)
}
})
}
}

7
website/docs/language/settings/backends/s3.mdx
View File

@@ -51,6 +51,11 @@ the target backend bucket:
* `s3:PutObject` on `arn:aws:s3:::mybucket/path/to/my/key`
* `s3:DeleteObject` on `arn:aws:s3:::mybucket/path/to/my/key`
OpenTofu may also need the following AWS IAM permissions on
the target backend bucket:
* `s3:PutObjectTagging` on `arn:aws:s3:::mybucket/path/to/my/key`
This is seen in the following AWS IAM Statement:
```json
@@ -342,6 +347,8 @@ The following configuration is required:
The following configuration is optional:
* `acl` - (Optional) [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) to be applied to the state file.
* `state_tags` - (Optional) [Tags](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) to be applied to the state object.
* `lock_tags` - (Optional) [Tags](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html) to be applied to the lock object.
* `encrypt` - (Optional) Enable [server side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html) of the state file.
* `endpoint` - (Optional) **Deprecated** Custom endpoint for the AWS S3 API. This can also be sourced from the `AWS_S3_ENDPOINT` environment variable.
* `force_path_style` - (Optional) **Deprecated** Enable path-style S3 URLs (`https://<HOST>/<BUCKET>` instead of `https://<BUCKET>.<HOST>`). Use `use_path_style` instead.