Use parallel errgroup to increase file copy speed (#3214)

Signed-off-by: Christian Mesh <christianmesh1@gmail.com>

CHANGELOG.md

@@ -35,6 +35,7 @@ ENHANCEMENTS:
 * Reduced the CPU and Memory overhead of managing large state files in OpenTofu. ([#3110](https://github.com/opentofu/opentofu/pull/3110))
   * These improvements are primarilly visible in projects with thousands of resources
 * Upgrade github.com/hashicorp/go-getter to v1.7.9 to fix [GO-2025-3892](https://pkg.go.dev/vuln/GO-2025-3892). ([#3227](https://github.com/opentofu/opentofu/pull/3227))
+* The module installer will copy files in parallel to improve performance of `init` ([#3214](https://github.com/opentofu/opentofu/pull/3214))
 
 BUG FIXES:
 

go.mod

@@ -104,6 +104,7 @@ require (
 	golang.org/x/mod v0.26.0
 	golang.org/x/net v0.43.0
 	golang.org/x/oauth2 v0.30.0
+	golang.org/x/sync v0.16.0
 	golang.org/x/sys v0.35.0
 	golang.org/x/term v0.34.0
 	golang.org/x/text v0.28.0
@@ -257,7 +258,6 @@ require (
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
-	golang.org/x/sync v0.16.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
 	golang.org/x/tools/go/expect v0.1.1-deprecated // indirect
 	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect

internal/copy/copy_dir.go

@@ -6,11 +6,14 @@
 package copy
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+
+	"golang.org/x/sync/errgroup"
 )
 
 // CopyDir recursively copies all of the files within the directory given in
@@ -44,6 +47,8 @@ func CopyDir(dst, src string) error {
 		return fmt.Errorf("failed to evaluate symlinks for source %q: %w", src, err)
 	}
 
+	var errg errgroup.Group
+
 	walkFn := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("error walking the path %q: %w", path, err)
@@ -65,13 +70,6 @@ func CopyDir(dst, src string) error {
 		// destination with the path without the src on it.
 		dstPath := filepath.Join(dst, path[len(src):])
 
-		// we don't want to try and copy the same file over itself.
-		if eq, err := SameFile(path, dstPath); err != nil {
-			return fmt.Errorf("failed to check if files are the same: %w", err)
-		} else if eq {
-			return nil
-		}
-
 		// If we have a directory, make that subdirectory, then continue
 		// the walk.
 		if info.IsDir() {
@@ -87,22 +85,35 @@ func CopyDir(dst, src string) error {
 			return nil
 		}
 
-		// If the current path is a symlink, recreate the symlink relative to
-		// the dst directory
-		if info.Mode()&os.ModeSymlink == os.ModeSymlink {
-			target, err := os.Readlink(path)
-			if err != nil {
-				return fmt.Errorf("failed to read symlink %q: %w", path, err)
+		errg.Go(func() error {
+
+			// we don't want to try and copy the same file over itself.
+			if eq, err := SameFile(path, dstPath); err != nil {
+				return fmt.Errorf("failed to check if files are the same: %w", err)
+			} else if eq {
+				return nil
 			}
 
-			if err := os.Symlink(target, dstPath); err != nil {
-				return fmt.Errorf("failed to create symlink %q: %w", dstPath, err)
+			// If the current path is a symlink, recreate the symlink relative to
+			// the dst directory
+			if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+				target, err := os.Readlink(path)
+				if err != nil {
+					return fmt.Errorf("failed to read symlink %q: %w", path, err)
+				}
+
+				if err := os.Symlink(target, dstPath); err != nil {
+					return fmt.Errorf("failed to create symlink %q: %w", dstPath, err)
+				}
+				return nil
 			}
-			return nil
-		}
-		return copyFile(dstPath, path, info.Mode())
+			return copyFile(dstPath, path, info.Mode())
+		})
+		return nil
 	}
-	return filepath.Walk(src, walkFn)
+	err = filepath.Walk(src, walkFn)
+	waitErr := errg.Wait()
+	return errors.Join(waitErr, err)
 }
 
 // copyFile copies the contents and mode of the file from src to dst.