chore!: send modules archive over the proto messages (#21398)

# What this does

Dynamic parameters caches the `./terraform/modules` directory for parameter usage. What this PR does is send over this archive to the provisioner when building workspaces.

This allow terraform to skip downloading modules from their registries, a step that takes seconds. 

<img width="1223" height="429" alt="Screenshot From 2025-12-29 12-57-52" src="https://github.com/user-attachments/assets/16066e0a-ac79-4296-819d-924f4b0418dc" />


# Wire protocol

The wire protocol reuses the same mechanism used to download the modules `provisoner -> coder`. It splits up large archives into multiple protobuf messages so larger archives can be sent under the message size limit.

# 🚨  Behavior Change (Breaking Change) 🚨 

**Before this PR** modules were downloaded on every workspace build. This means unpinned modules always fetched the latest version

**After this PR** modules are cached at template import time, and their versions are effectively pinned for all subsequent workspace builds.

provisionersdk/dataupload.go

@@ -0,0 +1,85 @@
+package provisionersdk
+
+import (
+	"io"
+
+	"golang.org/x/xerrors"
+
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
+)
+
+// HandleReceivingDataUpload can download a multi-part file from a proto stream.
+// The stream is expected to be closed by the caller.
+func HandleReceivingDataUpload(stream interface {
+	Recv() (*sdkproto.FileUpload, error)
+},
+) (*sdkproto.DataBuilder, error) {
+	var file *sdkproto.DataBuilder
+UploadFileStream:
+	for {
+		msg, err := stream.Recv()
+		if err != nil {
+			if xerrors.Is(err, io.EOF) {
+				// Do not return an EOF here, as it is a "retryable error" in the client context.
+				// This failure indicates the download stream was closed prematurely, and it is a
+				// fatal error.
+				return nil, xerrors.Errorf("stream closed before file download complete")
+			}
+			return nil, xerrors.Errorf("receive file download: %w", err)
+		}
+
+		switch typed := msg.Type.(type) {
+		case *sdkproto.FileUpload_Error:
+			return nil, xerrors.Errorf("download file: %s", typed.Error.Error)
+		case *sdkproto.FileUpload_DataUpload:
+			if file != nil {
+				return nil, xerrors.New("unexpected file download while waiting for file completion")
+			}
+
+			file, err = sdkproto.NewDataBuilder(&sdkproto.DataUpload{
+				UploadType: typed.DataUpload.UploadType,
+				DataHash:   typed.DataUpload.DataHash,
+				FileSize:   typed.DataUpload.FileSize,
+				Chunks:     typed.DataUpload.Chunks,
+			})
+			if err != nil {
+				return nil, xerrors.Errorf("unable to create file download: %w", err)
+			}
+
+			if file.IsDone() {
+				// If a file is 0 bytes, we can consider it done immediately.
+				// This should never really happen in practice, but we handle it gracefully.
+				break UploadFileStream
+			}
+		case *sdkproto.FileUpload_ChunkPiece:
+			if file == nil {
+				return nil, xerrors.New("unexpected chunk piece while waiting for file upload")
+			}
+
+			done, err := file.Add(&sdkproto.ChunkPiece{
+				Data:         typed.ChunkPiece.Data,
+				FullDataHash: typed.ChunkPiece.FullDataHash,
+				PieceIndex:   typed.ChunkPiece.PieceIndex,
+			})
+			if err != nil {
+				return nil, xerrors.Errorf("unable to add a chunk piece: %w", err)
+			}
+
+			if done {
+				break UploadFileStream
+			}
+		default:
+			// This should never happen
+			return nil, xerrors.Errorf("received unknown file upload message type: %T", msg.Type)
+		}
+	}
+
+	// This needs to be called again by the caller to retrieve the final payload.
+	// It is called here to do a hash check and ensure the file is correct.
+	_, err := file.Complete()
+	if err != nil {
+		return nil, xerrors.Errorf("complete file upload: %w", err)
+	}
+
+	return file, nil
+}

provisionersdk/serve.go

@@ -33,8 +33,15 @@ type ServeOptions struct {
 	Experiments         codersdk.Experiments
 }
 
+// InitRequest wraps the InitRequest proto with the module archive bytes, which
+// is downloaded by the SDK from the hash field in the InitRequest proto.
+type InitRequest struct {
+	*proto.InitRequest
+	ModuleArchive []byte
+}
+
 type Server interface {
-	Init(s *Session, r *proto.InitRequest, canceledOrComplete <-chan struct{}) *proto.InitComplete
+	Init(s *Session, r *InitRequest, canceledOrComplete <-chan struct{}) *proto.InitComplete
 	Parse(s *Session, r *proto.ParseRequest, canceledOrComplete <-chan struct{}) *proto.ParseComplete
 	Plan(s *Session, r *proto.PlanRequest, canceledOrComplete <-chan struct{}) *proto.PlanComplete
 	Apply(s *Session, r *proto.ApplyRequest, canceledOrComplete <-chan struct{}) *proto.ApplyComplete

provisionersdk/serve_test.go

@@ -149,7 +149,7 @@ var _ provisionersdk.Server = unimplementedServer{}
 
 type unimplementedServer struct{}
 
-func (unimplementedServer) Init(s *provisionersdk.Session, r *proto.InitRequest, canceledOrComplete <-chan struct{}) *proto.InitComplete {
+func (unimplementedServer) Init(s *provisionersdk.Session, r *provisionersdk.InitRequest, canceledOrComplete <-chan struct{}) *proto.InitComplete {
 	return &proto.InitComplete{}
 }
 

provisionersdk/session.go

@@ -125,6 +125,7 @@ func (s *Session) handleRequests() error {
 			if s.initialized {
 				return xerrors.New("cannot init more than once per session")
 			}
+
 			initResp, err := s.handleInitRequest(init, requests)
 			if err != nil {
 				return err
@@ -185,9 +186,47 @@ func (s *Session) handleRequests() error {
 	return nil
 }
 
+// fromChannel implements the `Recv` api using an underlying channel for
+// downloading files.
+type fromChannel struct {
+	requests <-chan *proto.Request
+}
+
+func (f *fromChannel) Recv() (*proto.FileUpload, error) {
+	next, ok := <-f.requests
+	if !ok {
+		return nil, xerrors.New("channel closed")
+	}
+
+	// Only file download messages are expected here.
+	file := next.GetFile()
+	if file == nil {
+		return nil, xerrors.Errorf("expected file upload")
+	}
+
+	return file, nil
+}
+
 func (s *Session) handleInitRequest(init *proto.InitRequest, requests <-chan *proto.Request) (*proto.InitComplete, error) {
-	r := &request[*proto.InitRequest, *proto.InitComplete]{
-		req:      init,
+	req := &InitRequest{
+		InitRequest:   init,
+		ModuleArchive: nil,
+	}
+	if len(init.GetInitialModuleTarHash()) > 0 {
+		file, err := HandleReceivingDataUpload(&fromChannel{requests: requests})
+		if err != nil {
+			return nil, err
+		}
+
+		data, err := file.Complete()
+		if err != nil {
+			return nil, err
+		}
+		req.ModuleArchive = data
+	}
+
+	r := &request[*InitRequest, *proto.InitComplete]{
+		req:      req,
 		session:  s,
 		serverFn: s.server.Init,
 		cancels:  requests,
@@ -279,7 +318,7 @@ func (s *Session) ProvisionLog(level proto.LogLevel, output string) {
 }
 
 type pRequest interface {
-	*proto.ParseRequest | *proto.InitRequest | *proto.PlanRequest | *proto.ApplyRequest | *proto.GraphRequest
+	*proto.ParseRequest | *InitRequest | *proto.PlanRequest | *proto.ApplyRequest | *proto.GraphRequest
 }
 
 type pComplete interface {

provisionersdk/tfpath/tfpath.go

@@ -72,17 +72,39 @@ func (l Layout) ModulesFilePath() string {
 	return filepath.Join(l.ModulesDirectory(), "modules.json")
 }
 
-func (l Layout) ExtractArchive(ctx context.Context, logger slog.Logger, fs afero.Fs, templateSourceArchive []byte) error {
-	logger.Info(ctx, "unpacking template source archive",
-		slog.F("size_bytes", len(templateSourceArchive)),
-	)
-
-	err := fs.MkdirAll(l.WorkDirectory(), 0o700)
+// ExtractArchive extracts the provided template source archive and modules archive into the working directory.
+// `modulesArchive` is optional and can be nil or empty.
+func (l Layout) ExtractArchive(ctx context.Context, logger slog.Logger, fs afero.Fs, templateSourceArchive, modulesArchive []byte) error {
+	err := extractArchive(ctx, logger, fs, l.WorkDirectory(), templateSourceArchive)
 	if err != nil {
-		return xerrors.Errorf("create work directory %q: %w", l.WorkDirectory(), err)
+		return xerrors.Errorf("extract template source archive: %w", err)
 	}
 
-	reader := tar.NewReader(bytes.NewBuffer(templateSourceArchive))
+	if len(modulesArchive) > 0 {
+		err = extractArchive(ctx, logger, fs, l.WorkDirectory(), modulesArchive)
+		if err != nil {
+			return xerrors.Errorf("extract modules archive: %w", err)
+		}
+	}
+	return nil
+}
+
+func isValidSessionDir(dirName string) bool {
+	match, err := filepath.Match(sessionDirPrefix+"*", dirName)
+	return err == nil && match
+}
+
+func extractArchive(ctx context.Context, logger slog.Logger, fs afero.Fs, directory string, archive []byte) error {
+	logger.Info(ctx, "unpacking source archive",
+		slog.F("size_bytes", len(archive)),
+	)
+
+	err := fs.MkdirAll(directory, 0o700)
+	if err != nil {
+		return xerrors.Errorf("create work directory %q: %w", directory, err)
+	}
+
+	reader := tar.NewReader(bytes.NewBuffer(archive))
 	for {
 		header, err := reader.Next()
 		if err != nil {
@@ -103,8 +125,8 @@ func (l Layout) ExtractArchive(ctx context.Context, logger slog.Logger, fs afero
 		}
 
 		// nolint: gosec // Safe to no-lint because the filepath.IsLocal check above.
-		headerPath := filepath.Join(l.WorkDirectory(), header.Name)
-		if !strings.HasPrefix(headerPath, filepath.Clean(l.WorkDirectory())) {
+		headerPath := filepath.Join(directory, header.Name)
+		if !strings.HasPrefix(headerPath, filepath.Clean(directory)) {
 			return xerrors.New("tar attempts to target relative upper directory")
 		}
 		mode := header.FileInfo().Mode()
@@ -220,8 +242,3 @@ func (l Layout) CleanStaleSessions(ctx context.Context, logger slog.Logger, fs a
 	}
 	return nil
 }
-
-func isValidSessionDir(dirName string) bool {
-	match, err := filepath.Match(sessionDirPrefix+"*", dirName)
-	return err == nil && match
-}