feat: add support for networked provisioners (#9593)

* Refactor provisionerd to use interface to connect to provisioners Signed-off-by: Spike Curtis <spike@coder.com> * feat: add support for networked provisioners Signed-off-by: Spike Curtis <spike@coder.com> * fix token length and linting Signed-off-by: Spike Curtis <spike@coder.com> --------- Signed-off-by: Spike Curtis <spike@coder.com>
2026-06-02 20:48:20 +00:00 · 2023-09-08 13:53:48 +04:00
parent 8b51a2f3c5
commit 11b6068112
11 changed files with 994 additions and 45 deletions
@@ -0,0 +1,28 @@
+package provisionerd
+
+import (
+	"context"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/provisionerd/proto"
+
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
+)
+
+// LocalProvisioners is a Connector that stores a static set of in-process
+// provisioners.
+type LocalProvisioners map[string]sdkproto.DRPCProvisionerClient
+
+func (l LocalProvisioners) Connect(_ context.Context, job *proto.AcquiredJob, respCh chan<- ConnectResponse) {
+	r := ConnectResponse{Job: job}
+	p, ok := l[job.Provisioner]
+	if ok {
+		r.Client = p
+	} else {
+		r.Error = xerrors.Errorf("missing provisioner type %s", job.Provisioner)
+	}
+	go func() {
+		respCh <- r
+	}()
+}
@@ -32,8 +32,24 @@ import (
 // Dialer represents the function to create a daemon client connection.
 type Dialer func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error)

-// Provisioners maps provisioner ID to implementation.
-type Provisioners map[string]sdkproto.DRPCProvisionerClient
+// ConnectResponse is the response returned asynchronously from Connector.Connect
+// containing either the Provisioner Client or an Error.  The Job is also returned
+// unaltered to disambiguate responses if the respCh is shared among multiple jobs
+type ConnectResponse struct {
+	Job    *proto.AcquiredJob
+	Client sdkproto.DRPCProvisionerClient
+	Error  error
+}
+
+// Connector allows the provisioner daemon to Connect to a provisioner
+// for the given job.
+type Connector interface {
+	// Connect to the correct provisioner for the given job. The response is
+	// delivered asynchronously over the respCh.  If the provided context expires,
+	// the Connector may stop waiting for the provisioner and return an error
+	// response.
+	Connect(ctx context.Context, job *proto.AcquiredJob, respCh chan<- ConnectResponse)
+}

 // Options provides customizations to the behavior of a provisioner daemon.
 type Options struct {
@@ -47,7 +63,7 @@ type Options struct {
 	JobPollInterval     time.Duration
 	JobPollJitter       time.Duration
 	JobPollDebounce     time.Duration
-	Provisioners        Provisioners
+	Connector           Connector
 }

 // New creates and starts a provisioner daemon.
@@ -375,11 +391,13 @@ func (p *Server) acquireJob(ctx context.Context) {

 	p.opts.Logger.Debug(ctx, "acquired job", fields...)

-	provisioner, ok := p.opts.Provisioners[job.Provisioner]
-	if !ok {
+	respCh := make(chan ConnectResponse)
+	p.opts.Connector.Connect(ctx, job, respCh)
+	resp := <-respCh
+	if resp.Error != nil {
 		err := p.FailJob(ctx, &proto.FailedJob{
 			JobId: job.JobId,
-			Error: fmt.Sprintf("no provisioner %s", job.Provisioner),
+			Error: fmt.Sprintf("failed to connect to provisioner: %s", resp.Error),
 		})
 		if err != nil {
 			p.opts.Logger.Error(ctx, "provisioner job failed", slog.F("job_id", job.JobId), slog.Error(err))
@@ -394,7 +412,7 @@ func (p *Server) acquireJob(ctx context.Context) {
 			Updater:             p,
 			QuotaCommitter:      p,
 			Logger:              p.opts.Logger.Named("runner"),
-			Provisioner:         provisioner,
+			Provisioner:         resp.Client,
 			UpdateInterval:      p.opts.UpdateInterval,
 			ForceCancelInterval: p.opts.ForceCancelInterval,
 			LogDebounceInterval: p.opts.LogBufferInterval,
@@ -60,7 +60,7 @@ func TestProvisionerd(t *testing.T) {
 		})
 		closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
 			return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{}), nil
-		}, provisionerd.Provisioners{})
+		}, provisionerd.LocalProvisioners{})
 		require.NoError(t, closer.Close())
 	})

@@ -74,7 +74,7 @@ func TestProvisionerd(t *testing.T) {
 		closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
 			defer close(completeChan)
 			return nil, xerrors.New("an error")
-		}, provisionerd.Provisioners{})
+		}, provisionerd.LocalProvisioners{})
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		require.NoError(t, closer.Close())
 	})
@@ -101,7 +101,7 @@ func TestProvisionerd(t *testing.T) {
 				},
 				updateJob: noopUpdateJob,
 			}), nil
-		}, provisionerd.Provisioners{})
+		}, provisionerd.LocalProvisioners{})
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
 		require.NoError(t, closer.Close())
 	})
@@ -141,7 +141,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				parse: func(_ *provisionersdk.Session, _ *sdkproto.ParseRequest, _ <-chan struct{}) *sdkproto.ParseComplete {
 					closerMutex.Lock()
@@ -195,7 +195,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{}),
 		})
 		require.Condition(t, closedWithin(completeChan, testutil.WaitShort))
@@ -237,7 +237,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				parse: func(
 					_ *provisionersdk.Session,
@@ -304,7 +304,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				parse: func(
 					s *provisionersdk.Session,
@@ -398,7 +398,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					_ *provisionersdk.Session,
@@ -472,7 +472,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -553,7 +553,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -638,7 +638,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -714,7 +714,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -800,7 +800,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -886,7 +886,7 @@ func TestProvisionerd(t *testing.T) {
 				}()
 			}
 			return client, nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					_ *provisionersdk.Session,
@@ -971,7 +971,7 @@ func TestProvisionerd(t *testing.T) {
 				}()
 			}
 			return client, nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					_ *provisionersdk.Session,
@@ -1055,7 +1055,7 @@ func TestProvisionerd(t *testing.T) {
 					return &proto.Empty{}, nil
 				},
 			}), nil
-		}, provisionerd.Provisioners{
+		}, provisionerd.LocalProvisioners{
 			"someprovisioner": createProvisionerClient(t, done, provisionerTestServer{
 				plan: func(
 					s *provisionersdk.Session,
@@ -1103,12 +1103,12 @@ func createTar(t *testing.T, files map[string]string) []byte {
 }

 // Creates a provisionerd implementation with the provided dialer and provisioners.
-func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, provisioners provisionerd.Provisioners) *provisionerd.Server {
+func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, connector provisionerd.LocalProvisioners) *provisionerd.Server {
 	server := provisionerd.New(dialer, &provisionerd.Options{
 		Logger:          slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Named("provisionerd").Leveled(slog.LevelDebug),
 		JobPollInterval: 50 * time.Millisecond,
 		UpdateInterval:  50 * time.Millisecond,
-		Provisioners:    provisioners,
+		Connector:       connector,
 	})
 	t.Cleanup(func() {
 		_ = server.Close()