jprdonnelly/opentf
1
0
Fork 0
mirror of https://github.com/opentffoundation/opentf.git synced 2025-12-25 01:00:16 -05:00
Code Issues Packages Projects Releases Wiki Activity

build: Build and run e2etest as part of the release build pipeline

Browse Source 
This uses the decoupled build and run strategy to run the e2etests so that
we can arrange to run the tests against the real release packages produced
elsewhere in this workflow, rather than ones generated just in time by
the test harness.

The modifications to make-archive.sh here make it more consistent with its
originally-intended purpose of producing a harness for testing "real"
release executables. Our earlier compromise of making it include its own
terraform executable came from a desire to use that script as part of
manual cross-platform testing when we weren't yet set up to support
automation of those tests as we're doing here. That does mean, however,
that the terraform-e2etest package content must be combined with content
from a terraform release package in order to produce a valid contest for
running the tests.

We use a single job to cross-compile the test harness for all of the
supported platforms, because that build is relatively fast and so not
worth the overhead of matrix build, but then use a matrix build to
actually run the tests so that we can run them in a worker matching the
target platform.

We currently have access only to amd64 (x64) runners in GitHub Actions
and so for the moment this process is limited only to the subset of our
supported platforms which use that architecture.
This commit is contained in:
Martin Atkins
2021-12-23 09:10:59 -08:00
parent 6704f8c795
commit b802db75d7
8 changed files with 269 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/command/e2etest/main_test.go
View File

@@ -11,6 +11,18 @@ import (
var terraformBin string
// canRunGoBuild is a short-term compromise to account for the fact that we
// have a small number of tests that work by building helper programs using
// "go build" at runtime, but we can't do that in our isolated test mode
// driven by the make-archive.sh script.
//
// FIXME: Rework this a bit so that we build the necessary helper programs
// (test plugins, etc) as part of the initial suite setup, and in the
// make-archive.sh script, so that we can run all of the tests in both
// situations with the tests just using the executable already built for
// them, as we do for terraformBin.
var canRunGoBuild bool
func TestMain(m *testing.M) {
teardown := setup()
code := m.Run()
@@ -21,10 +33,10 @@ func TestMain(m *testing.M) {
func setup() func() {
if terraformBin != "" {
// this is pre-set when we're running in a binary produced from
// the make-archive.sh script, since that builds a ready-to-go
// binary into the archive. However, we do need to turn it into
// an absolute path so that we can find it when we change the
// working directory during tests.
// the make-archive.sh script, since that is for testing an
// executable obtained from a real release package. However, we do
// need to turn it into an absolute path so that we can find it
// when we change the working directory during tests.
var err error
terraformBin, err = filepath.Abs(terraformBin)
if err != nil {
@@ -38,6 +50,11 @@ func setup() func() {
// Make the executable available for use in tests
terraformBin = tmpFilename
// Tests running in the ad-hoc testing mode are allowed to use "go build"
// and similar to produce other test executables.
// (See the comment on this variable's declaration for more information.)
canRunGoBuild = true
return func() {
os.Remove(tmpFilename)
}

13
internal/command/e2etest/make-archive.sh
View File

@@ -13,9 +13,12 @@
# and then executed as follows:
# set TF_ACC=1
# ./e2etest.exe
# Since the test archive includes both the test fixtures and the compiled
# terraform executable along with this test program, the result is
# self-contained and does not require a local Go compiler on the target system.
#
# Because separated e2etest harnesses are intended for testing against "real"
# release executables, the generated archives don't include a copy of
# the Terraform executable. Instead, the caller of the tests must retrieve
# and extract a release package into the working directory before running
# the e2etest executable, so that "e2etest" can find and execute it.
set +euo pipefail
@@ -33,10 +36,6 @@ mkdir -p "$OUTDIR"
# We need the test fixtures available when we run the tests.
cp -r testdata "$OUTDIR/testdata"
# Bundle a copy of our binary so the target system doesn't need the go
# compiler installed.
go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform
# Build the test program
go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/internal/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/internal/command/e2etest

4
internal/command/e2etest/primary_test.go
View File

@@ -204,13 +204,13 @@ func TestPrimaryChdirOption(t *testing.T) {
}
gotOutput := state.RootModule().OutputValues["cwd"]
wantOutputValue := cty.StringVal(tf.Path()) // path.cwd returns the original path, because path.root is how we get the overridden path
wantOutputValue := cty.StringVal(filepath.ToSlash(tf.Path())) // path.cwd returns the original path, because path.root is how we get the overridden path
if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
t.Errorf("incorrect value for cwd output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
}
gotOutput = state.RootModule().OutputValues["root"]
wantOutputValue = cty.StringVal(tf.Path("subdir")) // path.root is a relative path, but the text fixture uses abspath on it.
wantOutputValue = cty.StringVal(filepath.ToSlash(tf.Path("subdir"))) // path.root is a relative path, but the text fixture uses abspath on it.
if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) {
t.Errorf("incorrect value for root output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue)
}

8
internal/command/e2etest/provider_dev_test.go
View File

@@ -18,6 +18,14 @@ import (
// we normally do, so they can just overwrite the same local executable
// in-place to iterate faster.
func TestProviderDevOverrides(t *testing.T) {
if !canRunGoBuild {
// We're running in a separate-build-then-run context, so we can't
// currently execute this test which depends on being able to build
// new executable at runtime.
//
// (See the comment on canRunGoBuild's declaration for more information.)
t.Skip("can't run without building a new provider executable")
}
t.Parallel()
tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override")

8
internal/command/e2etest/provider_plugin_test.go
View File

@@ -13,6 +13,14 @@ import (
// TestProviderProtocols verifies that Terraform can execute provider plugins
// with both supported protocol versions.
func TestProviderProtocols(t *testing.T) {
if !canRunGoBuild {
// We're running in a separate-build-then-run context, so we can't
// currently execute this test which depends on being able to build
// new executable at runtime.
//
// (See the comment on canRunGoBuild's declaration for more information.)
t.Skip("can't run without building a new provider executable")
}
t.Parallel()
tf := e2e.NewBinary(terraformBin, "testdata/provider-plugin")

16
internal/command/e2etest/providers_tamper_test.go
View File

@@ -41,12 +41,16 @@ func TestProviderTampering(t *testing.T) {
seedDir := tf.WorkDir()
const providerVersion = "3.1.0" // must match the version in the fixture config
pluginDir := ".terraform/providers/registry.terraform.io/hashicorp/null/" + providerVersion + "/" + getproviders.CurrentPlatform.String()
pluginExe := pluginDir + "/terraform-provider-null_v" + providerVersion + "_x5"
pluginDir := filepath.Join(".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String())
pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5")
if getproviders.CurrentPlatform.OS == "windows" {
pluginExe += ".exe" // ugh
}
// filepath.Join here to make sure we get the right path separator
// for whatever OS we're running these tests on.
providerCacheDir := filepath.Join(".terraform", "providers")
t.Run("cache dir totally gone", func(t *testing.T) {
tf := e2e.NewBinary(terraformBin, seedDir)
defer tf.Close()
@@ -61,7 +65,7 @@ func TestProviderTampering(t *testing.T) {
if err == nil {
t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
}
if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
}
if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -128,7 +132,7 @@ func TestProviderTampering(t *testing.T) {
if err == nil {
t.Fatalf("unexpected plan success\nstdout:\n%s", stdout)
}
if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
}
if want := `terraform init`; !strings.Contains(stderr, want) {
@@ -237,7 +241,7 @@ func TestProviderTampering(t *testing.T) {
if err == nil {
t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
}
if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) {
if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) {
t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
}
})
@@ -260,7 +264,7 @@ func TestProviderTampering(t *testing.T) {
if err == nil {
t.Fatalf("unexpected apply success\nstdout:\n%s", stdout)
}
if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) {
t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr)
}
})

8
internal/command/e2etest/provisioner_plugin_test.go
View File

@@ -12,6 +12,14 @@ import (
// TestProvisionerPlugin is a test that terraform can execute a 3rd party
// provisioner plugin.
func TestProvisionerPlugin(t *testing.T) {
if !canRunGoBuild {
// We're running in a separate-build-then-run context, so we can't
// currently execute this test which depends on being able to build
// new executable at runtime.
//
// (See the comment on canRunGoBuild's declaration for more information.)
t.Skip("can't run without building a new provisioner executable")
}
t.Parallel()
// This test reaches out to releases.hashicorp.com to download the