modules: Start of decoding the declarations in a module

For now this is only for input variables, and only far enough to get their
names and detect duplicates. More to come in future commits.

Signed-off-by: Martin Atkins <mart@degeneration.co.uk>

internal/modules/by_example_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package modules
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/hashicorp/hcl/v2"
+
+	"github.com/opentofu/opentofu/internal/tfdiags"
+)
+
+// TestByExample attempts to load each of the modules in subdirectories of
+// testdata/by-example, expecting them to either be valid or to have expected
+// errors/warnings marked by end-of-line comments.
+//
+// There's more information on how this is intended to work in
+// testdata/by-example/README.
+//
+// This is intended only for broad regression testing, focused on whether
+// previously-valid things remain valid and previously-failing things
+// continue to fail with similar errors. Most language features should also be
+// covered by more specific unit tests, although it's okay to reuse some
+// of the modules under testdata/by-example for those more specific tests to
+// ease maintenence.
+func TestByExample(t *testing.T) {
+	baseDir := "testdata/by-example"
+	exampleDirs, err := os.ReadDir(baseDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, entry := range exampleDirs {
+		if !entry.IsDir() {
+			continue
+		}
+		t.Run(entry.Name(), func(t *testing.T) {
+			dir := filepath.Join(baseDir, entry.Name())
+			wantDiags := collectExpectedDiagnostics(t, dir)
+			_, gotDiags := LoadModuleFromDir(dir)
+			for _, got := range gotDiags {
+				want, ok := wantDiags.ExpectedDiagnostic(got)
+				if !ok {
+					t.Errorf("unexpected diagnostic: %s", spew.Sdump(got))
+					continue
+				}
+				if want.Severity != got.Severity() {
+					t.Errorf("wrong severity in diagnostic\nwant: %s\ngot: %s", want.Severity, spew.Sdump(got))
+				}
+				if want.Summary != got.Description().Summary {
+					t.Errorf("wrong summary in diagnostic\nwant: %s\ngot: %s", want.Summary, spew.Sdump(got))
+				}
+				// We remove our wantDiags entry for each match, so that
+				// any leftovers for the end are missing expected diagostics.
+				wantDiags.ForgetMatchingExpected(got)
+			}
+			for key, diag := range wantDiags {
+				t.Errorf("missing expected diagnostic\nwant: %s %s at %s:%d", diag.Severity, diag.Summary, key.Filename, key.Line)
+			}
+		})
+	}
+}
+
+type expectedDiagnostic struct {
+	Severity tfdiags.Severity
+	Summary  string
+}
+
+type expectedDiagnosticKey struct {
+	Filename string
+	Line     int
+}
+
+type expectedDiagnostics map[expectedDiagnosticKey]expectedDiagnostic
+
+func collectExpectedDiagnostics(t *testing.T, dir string) expectedDiagnostics {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("can't read %s: %s", dir, err)
+	}
+	ret := make(expectedDiagnostics)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+		filename := filepath.Join(dir, entry.Name())
+		src, err := os.ReadFile(filename)
+		if err != nil {
+			t.Fatalf("can't read %s: %s", filename, err)
+		}
+		sc := hcl.NewRangeScanner(src, filename, bufio.ScanLines)
+		for sc.Scan() {
+			const errorPrefix = "ERROR:"
+			const warningPrefix = "WARNING:"
+
+			line := string(sc.Bytes())
+			rng := sc.Range()
+			if idx := strings.Index(line, errorPrefix); idx != -1 {
+				wantSummary := strings.TrimSpace(line[idx+len(errorPrefix):])
+				key := expectedDiagnosticKey{rng.Filename, rng.Start.Line}
+				ret[key] = expectedDiagnostic{tfdiags.Error, wantSummary}
+			}
+			if idx := strings.Index(line, warningPrefix); idx != -1 {
+				wantSummary := strings.TrimSpace(line[idx+len(warningPrefix):])
+				key := expectedDiagnosticKey{rng.Filename, rng.Start.Line}
+				ret[key] = expectedDiagnostic{tfdiags.Warning, wantSummary}
+			}
+		}
+		if err := sc.Err(); err != nil {
+			t.Fatalf("can't read %s: %s", filename, err)
+		}
+	}
+	return ret
+}
+
+func (s expectedDiagnostics) ExpectedDiagnostic(diag tfdiags.Diagnostic) (expectedDiagnostic, bool) {
+	rng := diag.Source().Subject
+	if rng == nil {
+		return expectedDiagnostic{}, false // Sourceless diagnostics cannot possibly match
+	}
+	key := expectedDiagnosticKey{rng.Filename, rng.Start.Line}
+	expected, ok := s[key]
+	return expected, ok
+}
+
+func (s expectedDiagnostics) ForgetMatchingExpected(diag tfdiags.Diagnostic) {
+	rng := diag.Source().Subject
+	if rng == nil {
+		return
+	}
+	key := expectedDiagnosticKey{rng.Filename, rng.Start.Line}
+	delete(s, key)
+}

internal/modules/input_variable.go

@@ -0,0 +1,41 @@
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package modules
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+
+	"github.com/opentofu/opentofu/internal/tfdiags"
+)
+
+type InputVariable struct {
+	Name      string
+	DeclRange tfdiags.SourceRange
+}
+
+func decodeInputVariableBlock(block *hcl.Block) (*InputVariable, tfdiags.Diagnostics) {
+	var diags tfdiags.Diagnostics
+
+	ret := &InputVariable{
+		Name:      block.Labels[0],
+		DeclRange: tfdiags.SourceRangeFromHCL(block.DefRange),
+	}
+	if !hclsyntax.ValidIdentifier(ret.Name) {
+		diags = diags.Append(&hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid name for input variable",
+			Detail:   fmt.Sprintf("Cannot use %q as the name of an input variable. Name must contain only letters, digits, underscores, and dashes.", ret.Name),
+			Subject:  block.LabelRanges[0].Ptr(),
+		})
+	}
+
+	// TODO: Shallowly decode the arguments inside the block
+
+	return ret, diags
+}

internal/modules/module.go

@@ -32,9 +32,11 @@ import (
 // live in some other package and use different struct types.)
 type Module struct {
 	Dir string
+
+	InputVariables map[string]*InputVariable
 }
 
-func LoadModuleDir(dir string) (*Module, tfdiags.Diagnostics) {
+func LoadModuleFromDir(dir string) (*Module, tfdiags.Diagnostics) {
 	var diags tfdiags.Diagnostics
 	ret := &Module{
 		Dir: dir,
@@ -61,7 +63,63 @@ func LoadModuleDir(dir string) (*Module, tfdiags.Diagnostics) {
 		return ret, diags
 	}
 
-	// TODO: Decode everything else!
+	// Once we've confirmed that this module hasn't explicitly told us it
+	// is for a different version of OpenTofu, we can collect all of the
+	// diagnostics we encountered while loading the individual files, and
+	// bail out early if things were so invalid that we couldn't even
+	// complete shallow decoding. We delay this intentionally so that
+	// later OpenTofu versions could, for example, use laternewer HCL syntax
+	// features that cause recoverable parse errors without those blocking
+	// us from checking what OpenTofu versions the module was declared as
+	// being compatible with.
+	for _, file := range primaryFiles {
+		diags = diags.Append(file.Diagnostics)
+	}
+	for _, file := range overrideFiles {
+		diags = diags.Append(file.Diagnostics)
+	}
+	if diags.HasErrors() {
+		return ret, diags
+	}
+
+	// If we've got this far without encountering any errors, it's time to
+	// analyze one level deeper with the content of the top-level declarations.
+	ret.InputVariables = make(map[string]*InputVariable)
+	for _, file := range primaryFiles {
+		for _, block := range file.ConfigBlocks {
+			switch block.Type {
+			case "variable":
+				varDecl, moreDiags := decodeInputVariableBlock(block)
+				diags = diags.Append(moreDiags)
+				if varDecl == nil {
+					continue
+				}
+				if existing, exists := ret.InputVariables[varDecl.Name]; exists {
+					diags = diags.Append(&hcl.Diagnostic{
+						Severity: hcl.DiagError,
+						Summary:  "Duplicate input variable declaration",
+						Detail:   fmt.Sprintf("An input variable named %q was also declared at %s.", existing.Name, existing.DeclRange.StartString()),
+						Subject:  varDecl.DeclRange.ToHCL().Ptr(),
+					})
+					continue
+				}
+				ret.InputVariables[varDecl.Name] = varDecl
+			default:
+				// We should not get here because the cases above should
+				// cover all block types from [configFileSchema].
+				diags = diags.Append(fmt.Errorf("unhandled block type %q", block.Type))
+			}
+		}
+	}
+	for _, file := range overrideFiles {
+		// TODO: Implement something compatible with the existing merge
+		// behavior from "package configs".
+		diags = diags.Append(tfdiags.Sourceless(
+			tfdiags.Error,
+			"Override files not supported yet",
+			fmt.Sprintf("The prototype of a new approach to module loading does not yet handle override files, so cannot deal with %q.", file.Filename),
+		))
+	}
 
 	return ret, diags
 }

internal/modules/testdata/by-example/README

@@ -0,0 +1,32 @@
+The subdirectories of this directory each contain an "example" module that
+will be automatically tested by TestByExample in the "modules" package.
+
+All modules are expected to be loadable without generating any diagnostics,
+unless lines in the file are marked with end-of-line comments describing
+an expected diagnostic. For example:
+
+  vriable "example" { # ERROR: Unsupported block type
+
+  }
+
+A line containing "ERROR:" means that the example is expected to generate
+an error diagnostic referring to that line of source code, whose summary
+matches the text that follows "ERROR:" after stripping leading/trailing
+whitespace.
+
+A line containing "WARNING:" means the same except that the diagnostic is
+expected to have warning severity.
+
+Any diagnostic reported on a line that does not include one of these
+substrings is treated as unexpected and causes a test failure.
+
+By convention, examples that contain expected errors are named with the
+prefix "err-" to distinguish them from the examples that are expected to
+be valid (potentially with warnings). However, that prefix is for human
+convenience only.
+
+When an example directory contains only one source file, prefer to give it
+a basename that matches the directory name (ignoring "err-" prefix) so that
+it's easier to distinguish them in an editor with multiple examples open at
+once. When there are multiple files present, just try to select filenames that
+are all somehow related to the name of the directory containing the example.

internal/modules/testdata/by-example/err-unrecognized-block-type/unrecognized-block-type.tf

@@ -0,0 +1,2 @@
+not_valid { # ERROR: Unsupported block type
+}

internal/modules/testdata/by-example/variables/variables.tf

@@ -0,0 +1,30 @@
+# This example is a good place to add relatively-simple valid examples of
+# input variables where it seems like overkill to add an entirely new example
+# directory, but if you're adding a collection of example blocks related to
+# a specific feature then probably better to start a new directory to collect
+# those together thematically.
+
+variable "unconstrained" {
+}
+
+variable "string" {
+  type = string
+}
+
+variable "list_of_string" {
+  type = list(string)
+}
+
+variable "explicitly_unconstrained" {
+  type = any
+}
+
+variable "string_optional" {
+  type   = string
+  default = "hello"
+}
+
+variable "unconstrained_optional" {
+  type   = string
+  default = "hello"
+}

internal/modules/with_range.go

@@ -1,4 +1,9 @@
-package configs2
+// Copyright (c) The OpenTofu Authors
+// SPDX-License-Identifier: MPL-2.0
+// Copyright (c) 2023 HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package modules
 
 import (
 	"github.com/hashicorp/hcl/v2"