feat: track resource replacements when claiming a prebuilt workspace (#17571)

Closes https://github.com/coder/internal/issues/369

We can't know whether a replacement (i.e. drift of terraform state
leading to a resource needing to be deleted/recreated) will take place
apriori; we can only detect it at `plan` time, because the provider
decides whether a resource must be replaced and it cannot be inferred
through static analysis of the template.

**This is likely to be the most common gotcha with using prebuilds,
since it requires a slight template modification to use prebuilds
effectively**, so let's head this off before it's an issue for
customers.

Drift details will now be logged in the workspace build logs:


![image](https://github.com/user-attachments/assets/da1988b6-2cbe-4a79-a3c5-ea29891f3d6f)

Plus a notification will be sent to template admins when this situation
arises:


![image](https://github.com/user-attachments/assets/39d555b1-a262-4a3e-b529-03b9f23bf66a)

A new metric - `coderd_prebuilt_workspaces_resource_replacements_total`
- will also increment each time a workspace encounters replacements.

We only track _that_ a resource replacement occurred, not how many. Just
one is enough to ruin a prebuild, but we can't know apriori which
replacement would cause this.
For example, say we have 2 replacements: a `docker_container` and a
`null_resource`; we don't know which one might
cause an issue (or indeed if either would), so we just track the
replacement.

---------

Signed-off-by: Danny Kopping <dannykopping@gmail.com>

cli/server.go

@@ -928,6 +928,37 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 			options.StatsBatcher = batcher
 			defer closeBatcher()
 
+			// Manage notifications.
+			var (
+				notificationsCfg     = options.DeploymentValues.Notifications
+				notificationsManager *notifications.Manager
+			)
+
+			metrics := notifications.NewMetrics(options.PrometheusRegistry)
+			helpers := templateHelpers(options)
+
+			// The enqueuer is responsible for enqueueing notifications to the given store.
+			enqueuer, err := notifications.NewStoreEnqueuer(notificationsCfg, options.Database, helpers, logger.Named("notifications.enqueuer"), quartz.NewReal())
+			if err != nil {
+				return xerrors.Errorf("failed to instantiate notification store enqueuer: %w", err)
+			}
+			options.NotificationsEnqueuer = enqueuer
+
+			// The notification manager is responsible for:
+			//   - creating notifiers and managing their lifecycles (notifiers are responsible for dequeueing/sending notifications)
+			//   - keeping the store updated with status updates
+			notificationsManager, err = notifications.NewManager(notificationsCfg, options.Database, options.Pubsub, helpers, metrics, logger.Named("notifications.manager"))
+			if err != nil {
+				return xerrors.Errorf("failed to instantiate notification manager: %w", err)
+			}
+
+			// nolint:gocritic // We need to run the manager in a notifier context.
+			notificationsManager.Run(dbauthz.AsNotifier(ctx))
+
+			// Run report generator to distribute periodic reports.
+			notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
+			defer notificationReportGenerator.Close()
+
 			// We use a separate coderAPICloser so the Enterprise API
 			// can have its own close functions. This is cleaner
 			// than abstracting the Coder API itself.
@@ -975,37 +1006,6 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
 				return xerrors.Errorf("write config url: %w", err)
 			}
 
-			// Manage notifications.
-			var (
-				notificationsCfg     = options.DeploymentValues.Notifications
-				notificationsManager *notifications.Manager
-			)
-
-			metrics := notifications.NewMetrics(options.PrometheusRegistry)
-			helpers := templateHelpers(options)
-
-			// The enqueuer is responsible for enqueueing notifications to the given store.
-			enqueuer, err := notifications.NewStoreEnqueuer(notificationsCfg, options.Database, helpers, logger.Named("notifications.enqueuer"), quartz.NewReal())
-			if err != nil {
-				return xerrors.Errorf("failed to instantiate notification store enqueuer: %w", err)
-			}
-			options.NotificationsEnqueuer = enqueuer
-
-			// The notification manager is responsible for:
-			//   - creating notifiers and managing their lifecycles (notifiers are responsible for dequeueing/sending notifications)
-			//   - keeping the store updated with status updates
-			notificationsManager, err = notifications.NewManager(notificationsCfg, options.Database, options.Pubsub, helpers, metrics, logger.Named("notifications.manager"))
-			if err != nil {
-				return xerrors.Errorf("failed to instantiate notification manager: %w", err)
-			}
-
-			// nolint:gocritic // We need to run the manager in a notifier context.
-			notificationsManager.Run(dbauthz.AsNotifier(ctx))
-
-			// Run report generator to distribute periodic reports.
-			notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
-			defer notificationReportGenerator.Close()
-
 			// Since errCh only has one buffered slot, all routines
 			// sending on it must be wrapped in a select/default to
 			// avoid leaving dangling goroutines waiting for the

coderd/coderd.go

@@ -40,10 +40,11 @@ import (
 	"tailscale.com/util/singleflight"
 
 	"cdr.dev/slog"
-	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"
 
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
+
 	"github.com/coder/coder/v2/coderd/ai"
 	"github.com/coder/coder/v2/coderd/cryptokeys"
 	"github.com/coder/coder/v2/coderd/entitlements"
@@ -1795,6 +1796,7 @@ func (api *API) CreateInMemoryTaggedProvisionerDaemon(dialCtx context.Context, n
 			Clock:               api.Clock,
 		},
 		api.NotificationsEnqueuer,
+		&api.PrebuildsReconciler,
 	)
 	if err != nil {
 		return nil, err

coderd/database/migrations/000324_resource_replacements_notification.down.sql

@@ -0,0 +1 @@
+DELETE FROM notification_templates WHERE id = '89d9745a-816e-4695-a17f-3d0a229e2b8d';

coderd/database/migrations/000324_resource_replacements_notification.up.sql

@@ -0,0 +1,34 @@
+INSERT INTO notification_templates
+	(id, name, title_template, body_template, "group", actions)
+VALUES ('89d9745a-816e-4695-a17f-3d0a229e2b8d',
+		'Prebuilt Workspace Resource Replaced',
+		E'There might be a problem with a recently claimed prebuilt workspace',
+		$$
+Workspace **{{.Labels.workspace}}** was claimed from a prebuilt workspace by **{{.Labels.claimant}}**.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+{{range $resource, $paths := .Data.replacements -}}
+- _{{ $resource }}_  was replaced due to changes to _{{ $paths }}_
+{{end}}
+
+When Terraform must change an immutable attribute, it replaces the entire resource.
+If you’re using prebuilds to speed up provisioning, unexpected replacements will slow down
+workspace startup—even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the **{{.Labels.preset}}** preset.
+$$,
+		'Template Events',
+		'[
+		{
+			"label": "View workspace build",
+			"url": "{{base_url}}/@{{.Labels.claimant}}/{{.Labels.workspace}}/builds/{{.Labels.workspace_build_num}}"
+		},
+		{
+			"label": "View template version",
+			"url": "{{base_url}}/templates/{{.Labels.org}}/{{.Labels.template}}/versions/{{.Labels.template_version}}"
+		}
+	]'::jsonb);

coderd/notifications/events.go

@@ -39,6 +39,7 @@ var (
 	TemplateTemplateDeprecated = uuid.MustParse("f40fae84-55a2-42cd-99fa-b41c1ca64894")
 
 	TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")
+	TemplateWorkspaceResourceReplaced   = uuid.MustParse("89d9745a-816e-4695-a17f-3d0a229e2b8d")
 )
 
 // Notification-related events.

coderd/notifications/notifications_test.go

@@ -35,6 +35,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/quartz"
+	"github.com/coder/serpent"
+
 	"github.com/coder/coder/v2/coderd/coderdtest"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
@@ -48,8 +51,6 @@ import (
 	"github.com/coder/coder/v2/coderd/util/syncmap"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/testutil"
-	"github.com/coder/quartz"
-	"github.com/coder/serpent"
 )
 
 // updateGoldenFiles is a flag that can be set to update golden files.
@@ -1226,6 +1227,29 @@ func TestNotificationTemplates_Golden(t *testing.T) {
 				Labels:       map[string]string{},
 			},
 		},
+		{
+			name: "TemplateWorkspaceResourceReplaced",
+			id:   notifications.TemplateWorkspaceResourceReplaced,
+			payload: types.MessagePayload{
+				UserName:     "Bobby",
+				UserEmail:    "bobby@coder.com",
+				UserUsername: "bobby",
+				Labels: map[string]string{
+					"org":                 "cern",
+					"workspace":           "my-workspace",
+					"workspace_build_num": "2",
+					"template":            "docker",
+					"template_version":    "angry_torvalds",
+					"preset":              "particle-accelerator",
+					"claimant":            "prebuilds-claimer",
+				},
+				Data: map[string]any{
+					"replacements": map[string]string{
+						"docker_container[0]": "env, hostname",
+					},
+				},
+			},
+		},
 	}
 
 	// We must have a test case for every notification_template. This is enforced below:

coderd/notifications/notificationstest/fake_enqueuer.go

@@ -9,6 +9,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 )
@@ -19,6 +20,12 @@ type FakeEnqueuer struct {
 	sent       []*FakeNotification
 }
 
+var _ notifications.Enqueuer = &FakeEnqueuer{}
+
+func NewFakeEnqueuer() *FakeEnqueuer {
+	return &FakeEnqueuer{}
+}
+
 type FakeNotification struct {
 	UserID, TemplateID uuid.UUID
 	Labels             map[string]string

coderd/notifications/testdata/rendered-templates/smtp/TemplateWorkspaceResourceReplaced.html.golden

@@ -0,0 +1,131 @@
+From: system@coder.com
+To: bobby@coder.com
+Subject: There might be a problem with a recently claimed prebuilt workspace
+Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
+Date: Fri, 11 Oct 2024 09:03:06 +0000
+Content-Type: multipart/alternative;  boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+MIME-Version: 1.0
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/plain; charset=UTF-8
+
+Hi Bobby,
+
+Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-c=
+laimer.
+
+During the claim, Terraform destroyed and recreated the following resources
+because one or more immutable attributes changed:
+
+docker_container[0]  was replaced due to changes to env, hostname
+
+When Terraform must change an immutable attribute, it replaces the entire r=
+esource.
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down
+workspace startup=E2=80=94even when claiming a prebuilt environment.
+
+For tips on preventing replacements and improving claim performance, see th=
+is guide (https://coder.com/docs/admin/templates/extending-templates/prebui=
+lt-workspaces#preventing-resource-replacement).
+
+NOTE: this prebuilt workspace used the particle-accelerator preset.
+
+
+View workspace build: http://test.com/@prebuilds-claimer/my-workspace/build=
+s/2
+
+View template version: http://test.com/templates/cern/docker/versions/angry=
+_torvalds
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
+Content-Transfer-Encoding: quoted-printable
+Content-Type: text/html; charset=UTF-8
+
+<!doctype html>
+<html lang=3D"en">
+  <head>
+    <meta charset=3D"UTF-8" />
+    <meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
+=3D1.0" />
+    <title>There might be a problem with a recently claimed prebuilt worksp=
+ace</title>
+  </head>
+  <body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
+ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
+l', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; color: #020617=
+; background: #f8fafc;">
+    <div style=3D"max-width: 600px; margin: 20px auto; padding: 60px; borde=
+r: 1px solid #e2e8f0; border-radius: 8px; background-color: #fff; text-alig=
+n: left; font-size: 14px; line-height: 1.5;">
+      <div style=3D"text-align: center;">
+        <img src=3D"https://coder.com/coder-logo-horizontal.png" alt=3D"Cod=
+er Logo" style=3D"height: 40px;" />
+      </div>
+      <h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
+argin: 8px 0 32px; line-height: 1.5;">
+        There might be a problem with a recently claimed prebuilt workspace
+      </h1>
+      <div style=3D"line-height: 1.5;">
+        <p>Hi Bobby,</p>
+        <p>Workspace <strong>my-workspace</strong> was claimed from a prebu=
+ilt workspace by <strong>prebuilds-claimer</strong>.</p>
+
+<p>During the claim, Terraform destroyed and recreated the following resour=
+ces<br>
+because one or more immutable attributes changed:</p>
+
+<ul>
+<li>_docker<em>container[0]</em>  was replaced due to changes to <em>env, h=
+ostname</em><br>
+</li>
+</ul>
+
+<p>When Terraform must change an immutable attribute, it replaces the entir=
+e resource.<br>
+If you=E2=80=99re using prebuilds to speed up provisioning, unexpected repl=
+acements will slow down<br>
+workspace startup=E2=80=94even when claiming a prebuilt environment.</p>
+
+<p>For tips on preventing replacements and improving claim performance, see=
+ <a href=3D"https://coder.com/docs/admin/templates/extending-templates/preb=
+uilt-workspaces#preventing-resource-replacement">this guide</a>.</p>
+
+<p>NOTE: this prebuilt workspace used the <strong>particle-accelerator</str=
+ong> preset.</p>
+      </div>
+      <div style=3D"text-align: center; margin-top: 32px;">
+       =20
+        <a href=3D"http://test.com/@prebuilds-claimer/my-workspace/builds/2=
+" style=3D"display: inline-block; padding: 13px 24px; background-color: #02=
+0617; color: #f8fafc; text-decoration: none; border-radius: 8px; margin: 0 =
+4px;">
+          View workspace build
+        </a>
+       =20
+        <a href=3D"http://test.com/templates/cern/docker/versions/angry_tor=
+valds" style=3D"display: inline-block; padding: 13px 24px; background-color=
+: #020617; color: #f8fafc; text-decoration: none; border-radius: 8px; margi=
+n: 0 4px;">
+          View template version
+        </a>
+       =20
+      </div>
+      <div style=3D"border-top: 1px solid #e2e8f0; color: #475569; font-siz=
+e: 12px; margin-top: 64px; padding-top: 24px; line-height: 1.6;">
+        <p>&copy;&nbsp;2024&nbsp;Coder. All rights reserved&nbsp;-&nbsp;<a =
+href=3D"http://test.com" style=3D"color: #2563eb; text-decoration: none;">h=
+ttp://test.com</a></p>
+        <p><a href=3D"http://test.com/settings/notifications" style=3D"colo=
+r: #2563eb; text-decoration: none;">Click here to manage your notification =
+settings</a></p>
+        <p><a href=3D"http://test.com/settings/notifications?disabled=3D89d=
+9745a-816e-4695-a17f-3d0a229e2b8d" style=3D"color: #2563eb; text-decoration=
+: none;">Stop receiving emails like this</a></p>
+      </div>
+    </div>
+  </body>
+</html>
+
+--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4--

coderd/notifications/testdata/rendered-templates/webhook/TemplateWorkspaceResourceReplaced.json.golden

@@ -0,0 +1,42 @@
+{
+  "_version": "1.1",
+  "msg_id": "00000000-0000-0000-0000-000000000000",
+  "payload": {
+    "_version": "1.2",
+    "notification_name": "Prebuilt Workspace Resource Replaced",
+    "notification_template_id": "00000000-0000-0000-0000-000000000000",
+    "user_id": "00000000-0000-0000-0000-000000000000",
+    "user_email": "bobby@coder.com",
+    "user_name": "Bobby",
+    "user_username": "bobby",
+    "actions": [
+      {
+        "label": "View workspace build",
+        "url": "http://test.com/@prebuilds-claimer/my-workspace/builds/2"
+      },
+      {
+        "label": "View template version",
+        "url": "http://test.com/templates/cern/docker/versions/angry_torvalds"
+      }
+    ],
+    "labels": {
+      "claimant": "prebuilds-claimer",
+      "org": "cern",
+      "preset": "particle-accelerator",
+      "template": "docker",
+      "template_version": "angry_torvalds",
+      "workspace": "my-workspace",
+      "workspace_build_num": "2"
+    },
+    "data": {
+      "replacements": {
+        "docker_container[0]": "env, hostname"
+      }
+    },
+    "targets": null
+  },
+  "title": "There might be a problem with a recently claimed prebuilt workspace",
+  "title_markdown": "There might be a problem with a recently claimed prebuilt workspace",
+  "body": "Workspace my-workspace was claimed from a prebuilt workspace by prebuilds-claimer.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\ndocker_container[0]  was replaced due to changes to env, hostname\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see this guide (https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: this prebuilt workspace used the particle-accelerator preset.",
+  "body_markdown": "\nWorkspace **my-workspace** was claimed from a prebuilt workspace by **prebuilds-claimer**.\n\nDuring the claim, Terraform destroyed and recreated the following resources\nbecause one or more immutable attributes changed:\n\n- _docker_container[0]_  was replaced due to changes to _env, hostname_\n\n\nWhen Terraform must change an immutable attribute, it replaces the entire resource.\nIf you’re using prebuilds to speed up provisioning, unexpected replacements will slow down\nworkspace startup—even when claiming a prebuilt environment.\n\nFor tips on preventing replacements and improving claim performance, see [this guide](https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement).\n\nNOTE: this prebuilt workspace used the **particle-accelerator** preset.\n"
+}

coderd/prebuilds/api.go

@@ -7,6 +7,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )
 
 var (
@@ -27,6 +28,11 @@ type ReconciliationOrchestrator interface {
 	// Stop gracefully shuts down the orchestrator with the given cause.
 	// The cause is used for logging and error reporting.
 	Stop(ctx context.Context, cause error)
+
+	// TrackResourceReplacement handles a pathological situation whereby a terraform resource is replaced due to drift,
+	// which can obviate the whole point of pre-provisioning a prebuilt workspace.
+	// See more detail at https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement.
+	TrackResourceReplacement(ctx context.Context, workspaceID, buildID uuid.UUID, replacements []*sdkproto.ResourceReplacement)
 }
 
 type Reconciler interface {

coderd/prebuilds/noop.go

@@ -6,12 +6,15 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/coder/coder/v2/coderd/database"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 )
 
 type NoopReconciler struct{}
 
-func (NoopReconciler) Run(context.Context)                {}
+func (NoopReconciler) Run(context.Context)         {}
-func (NoopReconciler) Stop(context.Context, error)        {}
+func (NoopReconciler) Stop(context.Context, error) {}
+func (NoopReconciler) TrackResourceReplacement(context.Context, uuid.UUID, uuid.UUID, []*sdkproto.ResourceReplacement) {
+}
 func (NoopReconciler) ReconcileAll(context.Context) error { return nil }
 func (NoopReconciler) SnapshotState(context.Context, database.Store) (*GlobalSnapshot, error) {
 	return &GlobalSnapshot{}, nil

coderd/provisionerdserver/provisionerdserver.go

@@ -28,6 +28,7 @@ import (
 	protobuf "google.golang.org/protobuf/proto"
 
 	"cdr.dev/slog"
+
 	"github.com/coder/coder/v2/codersdk/drpcsdk"
 
 	"github.com/coder/quartz"
@@ -116,6 +117,7 @@ type server struct {
 	UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
 	DeploymentValues            *codersdk.DeploymentValues
 	NotificationsEnqueuer       notifications.Enqueuer
+	PrebuildsOrchestrator       *atomic.Pointer[prebuilds.ReconciliationOrchestrator]
 
 	OIDCConfig promoauth.OAuth2Config
 
@@ -151,8 +153,7 @@ func (t Tags) Valid() error {
 	return nil
 }
 
-func NewServer(
+func NewServer(lifecycleCtx context.Context,
-	lifecycleCtx context.Context,
 	accessURL *url.URL,
 	id uuid.UUID,
 	organizationID uuid.UUID,
@@ -171,6 +172,7 @@ func NewServer(
 	deploymentValues *codersdk.DeploymentValues,
 	options Options,
 	enqueuer notifications.Enqueuer,
+	prebuildsOrchestrator *atomic.Pointer[prebuilds.ReconciliationOrchestrator],
 ) (proto.DRPCProvisionerDaemonServer, error) {
 	// Fail-fast if pointers are nil
 	if lifecycleCtx == nil {
@@ -235,6 +237,7 @@ func NewServer(
 		acquireJobLongPollDur:       options.AcquireJobLongPollDur,
 		heartbeatInterval:           options.HeartbeatInterval,
 		heartbeatFn:                 options.HeartbeatFn,
+		PrebuildsOrchestrator:       prebuildsOrchestrator,
 	}
 
 	if s.heartbeatFn == nil {
@@ -1828,6 +1831,15 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
 			})
 		}
 
+		if s.PrebuildsOrchestrator != nil {
+			// Track resource replacements, if there are any.
+			orchestrator := s.PrebuildsOrchestrator.Load()
+			if resourceReplacements := completed.GetWorkspaceBuild().GetResourceReplacements(); orchestrator != nil && len(resourceReplacements) > 0 {
+				// Fire and forget. Bind to the lifecycle of the server so shutdowns are handled gracefully.
+				go (*orchestrator).TrackResourceReplacement(s.lifecycleCtx, workspace.ID, workspaceBuild.ID, resourceReplacements)
+			}
+		}
+
 		msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 			Kind:        wspubsub.WorkspaceEventKindStateChange,
 			WorkspaceID: workspace.ID,

coderd/provisionerdserver/provisionerdserver_test.go

@@ -26,11 +26,6 @@ import (
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"
 
-	"github.com/coder/coder/v2/coderd/prebuilds"
-	"github.com/coder/coder/v2/coderd/provisionerdserver"
-	"github.com/coder/coder/v2/coderd/rbac"
-	"github.com/coder/coder/v2/codersdk/agentsdk"
-
 	"github.com/coder/coder/v2/buildinfo"
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
@@ -42,11 +37,15 @@ import (
 	"github.com/coder/coder/v2/coderd/externalauth"
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/notifications/notificationstest"
+	agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
+	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/schedule"
 	"github.com/coder/coder/v2/coderd/schedule/cron"
 	"github.com/coder/coder/v2/coderd/telemetry"
 	"github.com/coder/coder/v2/coderd/wspubsub"
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
 	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
@@ -1889,7 +1888,7 @@ func TestCompleteJob(t *testing.T) {
 
 				// GIVEN something is listening to process workspace reinitialization:
 				reinitChan := make(chan agentsdk.ReinitializationEvent, 1) // Buffered to simplify test structure
-				cancel, err := prebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
+				cancel, err := agplprebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
 				require.NoError(t, err)
 				defer cancel()
 
@@ -1917,6 +1916,106 @@ func TestCompleteJob(t *testing.T) {
 			})
 		}
 	})
+
+	t.Run("PrebuiltWorkspaceClaimWithResourceReplacements", func(t *testing.T) {
+		t.Parallel()
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+
+		// Given: a mock prebuild orchestrator which stores calls to TrackResourceReplacement.
+		done := make(chan struct{})
+		orchestrator := &mockPrebuildsOrchestrator{
+			ReconciliationOrchestrator: agplprebuilds.DefaultReconciler,
+			done:                       done,
+		}
+		srv, db, ps, pd := setup(t, false, &overrides{
+			prebuildsOrchestrator: orchestrator,
+		})
+
+		// Given: a workspace build which simulates claiming a prebuild.
+		user := dbgen.User(t, db, database.User{})
+		template := dbgen.Template(t, db, database.Template{
+			Name:           "template",
+			Provisioner:    database.ProvisionerTypeEcho,
+			OrganizationID: pd.OrganizationID,
+		})
+		file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
+		workspaceTable := dbgen.Workspace(t, db, database.WorkspaceTable{
+			TemplateID:     template.ID,
+			OwnerID:        user.ID,
+			OrganizationID: pd.OrganizationID,
+		})
+		version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+			OrganizationID: pd.OrganizationID,
+			TemplateID: uuid.NullUUID{
+				UUID:  template.ID,
+				Valid: true,
+			},
+			JobID: uuid.New(),
+		})
+		build := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+			WorkspaceID:       workspaceTable.ID,
+			InitiatorID:       user.ID,
+			TemplateVersionID: version.ID,
+			Transition:        database.WorkspaceTransitionStart,
+			Reason:            database.BuildReasonInitiator,
+		})
+		job := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
+			FileID:      file.ID,
+			InitiatorID: user.ID,
+			Type:        database.ProvisionerJobTypeWorkspaceBuild,
+			Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
+				WorkspaceBuildID:            build.ID,
+				PrebuiltWorkspaceBuildStage: sdkproto.PrebuiltWorkspaceBuildStage_CLAIM,
+			})),
+			OrganizationID: pd.OrganizationID,
+		})
+		_, err := db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{
+			OrganizationID: pd.OrganizationID,
+			WorkerID: uuid.NullUUID{
+				UUID:  pd.ID,
+				Valid: true,
+			},
+			Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
+		})
+		require.NoError(t, err)
+
+		// When: a replacement is encountered.
+		replacements := []*sdkproto.ResourceReplacement{
+			{
+				Resource: "docker_container[0]",
+				Paths:    []string{"env"},
+			},
+		}
+
+		// Then: CompleteJob makes a call to TrackResourceReplacement.
+		_, err = srv.CompleteJob(ctx, &proto.CompletedJob{
+			JobId: job.ID.String(),
+			Type: &proto.CompletedJob_WorkspaceBuild_{
+				WorkspaceBuild: &proto.CompletedJob_WorkspaceBuild{
+					State:                []byte{},
+					ResourceReplacements: replacements,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		// Then: the replacements are as we expected.
+		testutil.RequireReceive(ctx, t, done)
+		require.Equal(t, replacements, orchestrator.replacements)
+	})
+}
+
+type mockPrebuildsOrchestrator struct {
+	agplprebuilds.ReconciliationOrchestrator
+
+	replacements []*sdkproto.ResourceReplacement
+	done         chan struct{}
+}
+
+func (m *mockPrebuildsOrchestrator) TrackResourceReplacement(_ context.Context, _, _ uuid.UUID, replacements []*sdkproto.ResourceReplacement) {
+	m.replacements = replacements
+	m.done <- struct{}{}
 }
 
 func TestInsertWorkspacePresetsAndParameters(t *testing.T) {
@@ -2803,6 +2902,7 @@ type overrides struct {
 	heartbeatInterval           time.Duration
 	auditor                     audit.Auditor
 	notificationEnqueuer        notifications.Enqueuer
+	prebuildsOrchestrator       agplprebuilds.ReconciliationOrchestrator
 }
 
 func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisionerDaemonServer, database.Store, pubsub.Pubsub, database.ProvisionerDaemon) {
@@ -2884,6 +2984,13 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
 	})
 	require.NoError(t, err)
 
+	prebuildsOrchestrator := ov.prebuildsOrchestrator
+	if prebuildsOrchestrator == nil {
+		prebuildsOrchestrator = agplprebuilds.DefaultReconciler
+	}
+	var op atomic.Pointer[agplprebuilds.ReconciliationOrchestrator]
+	op.Store(&prebuildsOrchestrator)
+
 	srv, err := provisionerdserver.NewServer(
 		ov.ctx,
 		&url.URL{},
@@ -2911,6 +3018,7 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
 			HeartbeatFn:           ov.heartbeatFn,
 		},
 		notifEnq,
+		&op,
 	)
 	require.NoError(t, err)
 	return srv, db, ps, daemon

docs/admin/templates/extending-templates/prebuilt-workspaces.md

@@ -163,6 +163,12 @@ resource "docker_container" "workspace" {
 
 Learn more about `ignore_changes` in the [Terraform documentation](https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle#ignore_changes).
 
+_A note on "immutable" attributes: Terraform providers may specify `ForceNew` on their resources' attributes. Any change
+to these attributes require the replacement (destruction and recreation) of the managed resource instance, rather than an in-place update.
+For example, the [`ami`](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance#ami-1) attribute on the `aws_instance` resource
+has [`ForceNew`](https://github.com/hashicorp/terraform-provider-aws/blob/main/internal/service/ec2/ec2_instance.go#L75-L81) set,
+since the AMI cannot be changed in-place._
+
 ### Current limitations
 
 The prebuilt workspaces feature has these current limitations:

enterprise/coderd/coderd.go

@@ -1165,6 +1165,6 @@ func (api *API) setupPrebuilds(featureEnabled bool) (agplprebuilds.Reconciliatio
 	}
 
 	reconciler := prebuilds.NewStoreReconciler(api.Database, api.Pubsub, api.DeploymentValues.Prebuilds,
-		api.Logger.Named("prebuilds"), quartz.NewReal(), api.PrometheusRegistry)
+		api.Logger.Named("prebuilds"), quartz.NewReal(), api.PrometheusRegistry, api.NotificationsEnqueuer)
 	return reconciler, prebuilds.NewEnterpriseClaimer(api.Database)
 }

enterprise/coderd/prebuilds/claim_test.go

@@ -147,7 +147,7 @@ func TestClaimPrebuild(t *testing.T) {
 				EntitlementsUpdateInterval: time.Second,
 			})
 
-			reconciler := prebuilds.NewStoreReconciler(spy, pubsub, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry())
+			reconciler := prebuilds.NewStoreReconciler(spy, pubsub, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 			var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(spy)
 			api.AGPL.PrebuildsClaimer.Store(&claimer)
 

enterprise/coderd/prebuilds/metricscollector.go

@@ -3,6 +3,7 @@ package prebuilds
 import (
 	"context"
 	"fmt"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -16,50 +17,73 @@ import (
 	"github.com/coder/coder/v2/coderd/prebuilds"
 )
 
+const (
+	namespace = "coderd_prebuilt_workspaces_"
+
+	MetricCreatedCount              = namespace + "created_total"
+	MetricFailedCount               = namespace + "failed_total"
+	MetricClaimedCount              = namespace + "claimed_total"
+	MetricResourceReplacementsCount = namespace + "resource_replacements_total"
+	MetricDesiredGauge              = namespace + "desired"
+	MetricRunningGauge              = namespace + "running"
+	MetricEligibleGauge             = namespace + "eligible"
+	MetricLastUpdatedGauge          = namespace + "metrics_last_updated"
+)
+
 var (
 	labels               = []string{"template_name", "preset_name", "organization_name"}
 	createdPrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_created_total",
+		MetricCreatedCount,
 		"Total number of prebuilt workspaces that have been created to meet the desired instance count of each "+
 			"template preset.",
 		labels,
 		nil,
 	)
 	failedPrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_failed_total",
+		MetricFailedCount,
 		"Total number of prebuilt workspaces that failed to build.",
 		labels,
 		nil,
 	)
 	claimedPrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_claimed_total",
+		MetricClaimedCount,
 		"Total number of prebuilt workspaces which were claimed by users. Claiming refers to creating a workspace "+
 			"with a preset selected for which eligible prebuilt workspaces are available and one is reassigned to a user.",
 		labels,
 		nil,
 	)
+	resourceReplacementsDesc = prometheus.NewDesc(
+		MetricResourceReplacementsCount,
+		"Total number of prebuilt workspaces whose resource(s) got replaced upon being claimed. "+
+			"In Terraform, drift on immutable attributes results in resource replacement. "+
+			"This represents a worst-case scenario for prebuilt workspaces because the pre-provisioned resource "+
+			"would have been recreated when claiming, thus obviating the point of pre-provisioning. "+
+			"See https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement",
+		labels,
+		nil,
+	)
 	desiredPrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_desired",
+		MetricDesiredGauge,
 		"Target number of prebuilt workspaces that should be available for each template preset.",
 		labels,
 		nil,
 	)
 	runningPrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_running",
+		MetricRunningGauge,
 		"Current number of prebuilt workspaces that are in a running state. These workspaces have started "+
 			"successfully but may not yet be claimable by users (see coderd_prebuilt_workspaces_eligible).",
 		labels,
 		nil,
 	)
 	eligiblePrebuildsDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_eligible",
+		MetricEligibleGauge,
 		"Current number of prebuilt workspaces that are eligible to be claimed by users. These are workspaces that "+
 			"have completed their build process with their agent reporting 'ready' status.",
 		labels,
 		nil,
 	)
 	lastUpdateDesc = prometheus.NewDesc(
-		"coderd_prebuilt_workspaces_metrics_last_updated",
+		MetricLastUpdatedGauge,
 		"The unix timestamp when the metrics related to prebuilt workspaces were last updated; these metrics are cached.",
 		[]string{},
 		nil,
@@ -77,6 +101,9 @@ type MetricsCollector struct {
 	snapshotter prebuilds.StateSnapshotter
 
 	latestState atomic.Pointer[metricsState]
+
+	replacementsCounter   map[replacementKey]float64
+	replacementsCounterMu sync.Mutex
 }
 
 var _ prometheus.Collector = new(MetricsCollector)
@@ -84,9 +111,10 @@ var _ prometheus.Collector = new(MetricsCollector)
 func NewMetricsCollector(db database.Store, logger slog.Logger, snapshotter prebuilds.StateSnapshotter) *MetricsCollector {
 	log := logger.Named("prebuilds_metrics_collector")
 	return &MetricsCollector{
-		database:    db,
+		database:            db,
-		logger:      log,
+		logger:              log,
-		snapshotter: snapshotter,
+		snapshotter:         snapshotter,
+		replacementsCounter: make(map[replacementKey]float64),
 	}
 }
 
@@ -94,6 +122,7 @@ func (*MetricsCollector) Describe(descCh chan<- *prometheus.Desc) {
 	descCh <- createdPrebuildsDesc
 	descCh <- failedPrebuildsDesc
 	descCh <- claimedPrebuildsDesc
+	descCh <- resourceReplacementsDesc
 	descCh <- desiredPrebuildsDesc
 	descCh <- runningPrebuildsDesc
 	descCh <- eligiblePrebuildsDesc
@@ -117,6 +146,12 @@ func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
 		metricsCh <- prometheus.MustNewConstMetric(claimedPrebuildsDesc, prometheus.CounterValue, float64(metric.ClaimedCount), metric.TemplateName, metric.PresetName, metric.OrganizationName)
 	}
 
+	mc.replacementsCounterMu.Lock()
+	for key, val := range mc.replacementsCounter {
+		metricsCh <- prometheus.MustNewConstMetric(resourceReplacementsDesc, prometheus.CounterValue, val, key.templateName, key.presetName, key.orgName)
+	}
+	mc.replacementsCounterMu.Unlock()
+
 	for _, preset := range currentState.snapshot.Presets {
 		if !preset.UsingActiveVersion {
 			continue
@@ -187,3 +222,24 @@ func (mc *MetricsCollector) UpdateState(ctx context.Context, timeout time.Durati
 	})
 	return nil
 }
+
+type replacementKey struct {
+	orgName, templateName, presetName string
+}
+
+func (k replacementKey) String() string {
+	return fmt.Sprintf("%s:%s:%s", k.orgName, k.templateName, k.presetName)
+}
+
+func (mc *MetricsCollector) trackResourceReplacement(orgName, templateName, presetName string) {
+	mc.replacementsCounterMu.Lock()
+	defer mc.replacementsCounterMu.Unlock()
+
+	key := replacementKey{orgName: orgName, templateName: templateName, presetName: presetName}
+
+	// We only track _that_ a resource replacement occurred, not how many.
+	// Just one is enough to ruin a prebuild, but we can't know apriori which replacement would cause this.
+	// For example, say we have 2 replacements: a docker_container and a null_resource; we don't know which one might
+	// cause an issue (or indeed if either would), so we just track the replacement.
+	mc.replacementsCounter[key]++
+}

enterprise/coderd/prebuilds/metricscollector_test.go

@@ -57,12 +57,12 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_claimed_total", ptr.To(0.0), true},
+				{prebuilds.MetricClaimedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_failed_total", ptr.To(0.0), true},
+				{prebuilds.MetricFailedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(0.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(0.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -74,12 +74,12 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_claimed_total", ptr.To(0.0), true},
+				{prebuilds.MetricClaimedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_failed_total", ptr.To(0.0), true},
+				{prebuilds.MetricFailedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(1.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -91,11 +91,11 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID, uuid.New()},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_failed_total", ptr.To(1.0), true},
+				{prebuilds.MetricFailedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(0.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(0.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -107,12 +107,12 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_claimed_total", ptr.To(0.0), true},
+				{prebuilds.MetricClaimedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_failed_total", ptr.To(0.0), true},
+				{prebuilds.MetricFailedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(1.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(1.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(1.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{true},
@@ -124,12 +124,12 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_claimed_total", ptr.To(0.0), true},
+				{prebuilds.MetricClaimedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_failed_total", ptr.To(0.0), true},
+				{prebuilds.MetricFailedCount, ptr.To(0.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(1.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -141,11 +141,11 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{uuid.New()},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_created_total", ptr.To(1.0), true},
+				{prebuilds.MetricCreatedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_claimed_total", ptr.To(1.0), true},
+				{prebuilds.MetricClaimedCount, ptr.To(1.0), true},
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(0.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(0.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -157,9 +157,9 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{uuid.New()},
 			ownerIDs:     []uuid.UUID{uuid.New()},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_desired", ptr.To(1.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_running", ptr.To(0.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(0.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{false},
 			eligible:        []bool{false},
@@ -171,7 +171,7 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID, uuid.New()},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_desired", ptr.To(0.0), false},
+				{prebuilds.MetricDesiredGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{true},
 			eligible:        []bool{false},
@@ -183,8 +183,8 @@ func TestMetricsCollector(t *testing.T) {
 			initiatorIDs: []uuid.UUID{agplprebuilds.SystemUserID},
 			ownerIDs:     []uuid.UUID{agplprebuilds.SystemUserID},
 			metrics: []metricCheck{
-				{"coderd_prebuilt_workspaces_running", ptr.To(1.0), false},
+				{prebuilds.MetricRunningGauge, ptr.To(1.0), false},
-				{"coderd_prebuilt_workspaces_eligible", ptr.To(0.0), false},
+				{prebuilds.MetricEligibleGauge, ptr.To(0.0), false},
 			},
 			templateDeleted: []bool{true},
 			eligible:        []bool{false},
@@ -220,7 +220,7 @@ func TestMetricsCollector(t *testing.T) {
 									})
 									clock := quartz.NewMock(t)
 									db, pubsub := dbtestutil.NewDB(t)
-									reconciler := prebuilds.NewStoreReconciler(db, pubsub, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry())
+									reconciler := prebuilds.NewStoreReconciler(db, pubsub, codersdk.PrebuildsConfig{}, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 									ctx := testutil.Context(t, testutil.WaitLong)
 
 									createdUsers := []uuid.UUID{agplprebuilds.SystemUserID}
@@ -242,7 +242,7 @@ func TestMetricsCollector(t *testing.T) {
 										org, template := setupTestDBTemplate(t, db, ownerID, templateDeleted)
 										templateVersionID := setupTestDBTemplateVersion(ctx, t, clock, db, pubsub, org.ID, ownerID, template.ID)
 										preset := setupTestDBPreset(t, db, templateVersionID, 1, uuid.New().String())
-										workspace := setupTestDBWorkspace(
+										workspace, _ := setupTestDBWorkspace(
 											t, clock, db, pubsub,
 											transition, jobStatus, org.ID, preset, template.ID, templateVersionID, initiatorID, ownerID,
 										)

enterprise/coderd/prebuilds/reconcile.go

@@ -3,8 +3,10 @@ package prebuilds
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"math"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -19,11 +21,13 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/provisionerjobs"
 	"github.com/coder/coder/v2/coderd/database/pubsub"
+	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
 	"github.com/coder/coder/v2/codersdk"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 
 	"cdr.dev/slog"
 
@@ -40,6 +44,7 @@ type StoreReconciler struct {
 	clock      quartz.Clock
 	registerer prometheus.Registerer
 	metrics    *MetricsCollector
+	notifEnq   notifications.Enqueuer
 
 	cancelFn          context.CancelCauseFunc
 	running           atomic.Bool
@@ -56,6 +61,7 @@ func NewStoreReconciler(store database.Store,
 	logger slog.Logger,
 	clock quartz.Clock,
 	registerer prometheus.Registerer,
+	notifEnq notifications.Enqueuer,
 ) *StoreReconciler {
 	reconciler := &StoreReconciler{
 		store:             store,
@@ -64,6 +70,7 @@ func NewStoreReconciler(store database.Store,
 		cfg:               cfg,
 		clock:             clock,
 		registerer:        registerer,
+		notifEnq:          notifEnq,
 		done:              make(chan struct{}, 1),
 		provisionNotifyCh: make(chan database.ProvisionerJob, 10),
 	}
@@ -633,3 +640,124 @@ func (c *StoreReconciler) provision(
 
 	return nil
 }
+
+// ForceMetricsUpdate forces the metrics collector, if defined, to update its state (we cache the metrics state to
+// reduce load on the database).
+func (c *StoreReconciler) ForceMetricsUpdate(ctx context.Context) error {
+	if c.metrics == nil {
+		return nil
+	}
+
+	return c.metrics.UpdateState(ctx, time.Second*10)
+}
+
+func (c *StoreReconciler) TrackResourceReplacement(ctx context.Context, workspaceID, buildID uuid.UUID, replacements []*sdkproto.ResourceReplacement) {
+	// nolint:gocritic // Necessary to query all the required data.
+	ctx = dbauthz.AsSystemRestricted(ctx)
+	// Since this may be called in a fire-and-forget fashion, we need to give up at some point.
+	trackCtx, trackCancel := context.WithTimeout(ctx, time.Minute)
+	defer trackCancel()
+
+	if err := c.trackResourceReplacement(trackCtx, workspaceID, buildID, replacements); err != nil {
+		c.logger.Error(ctx, "failed to track resource replacement", slog.Error(err))
+	}
+}
+
+// nolint:revive // Shut up it's fine.
+func (c *StoreReconciler) trackResourceReplacement(ctx context.Context, workspaceID, buildID uuid.UUID, replacements []*sdkproto.ResourceReplacement) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+
+	workspace, err := c.store.GetWorkspaceByID(ctx, workspaceID)
+	if err != nil {
+		return xerrors.Errorf("fetch workspace %q: %w", workspaceID.String(), err)
+	}
+
+	build, err := c.store.GetWorkspaceBuildByID(ctx, buildID)
+	if err != nil {
+		return xerrors.Errorf("fetch workspace build %q: %w", buildID.String(), err)
+	}
+
+	// The first build will always be the prebuild.
+	prebuild, err := c.store.GetWorkspaceBuildByWorkspaceIDAndBuildNumber(ctx, database.GetWorkspaceBuildByWorkspaceIDAndBuildNumberParams{
+		WorkspaceID: workspaceID, BuildNumber: 1,
+	})
+	if err != nil {
+		return xerrors.Errorf("fetch prebuild: %w", err)
+	}
+
+	// This should not be possible, but defend against it.
+	if !prebuild.TemplateVersionPresetID.Valid || prebuild.TemplateVersionPresetID.UUID == uuid.Nil {
+		return xerrors.Errorf("no preset used in prebuild for workspace %q", workspaceID.String())
+	}
+
+	prebuildPreset, err := c.store.GetPresetByID(ctx, prebuild.TemplateVersionPresetID.UUID)
+	if err != nil {
+		return xerrors.Errorf("fetch template preset for template version ID %q: %w", prebuild.TemplateVersionID.String(), err)
+	}
+
+	claimant, err := c.store.GetUserByID(ctx, workspace.OwnerID) // At this point, the workspace is owned by the new owner.
+	if err != nil {
+		return xerrors.Errorf("fetch claimant %q: %w", workspace.OwnerID.String(), err)
+	}
+
+	// Use the claiming build here (not prebuild) because both should be equivalent, and we might as well spot inconsistencies now.
+	templateVersion, err := c.store.GetTemplateVersionByID(ctx, build.TemplateVersionID)
+	if err != nil {
+		return xerrors.Errorf("fetch template version %q: %w", build.TemplateVersionID.String(), err)
+	}
+
+	org, err := c.store.GetOrganizationByID(ctx, workspace.OrganizationID)
+	if err != nil {
+		return xerrors.Errorf("fetch org %q: %w", workspace.OrganizationID.String(), err)
+	}
+
+	// Track resource replacement in Prometheus metric.
+	if c.metrics != nil {
+		c.metrics.trackResourceReplacement(org.Name, workspace.TemplateName, prebuildPreset.Name)
+	}
+
+	// Send notification to template admins.
+	if c.notifEnq == nil {
+		c.logger.Warn(ctx, "notification enqueuer not set, cannot send resource replacement notification(s)")
+		return nil
+	}
+
+	repls := make(map[string]string, len(replacements))
+	for _, repl := range replacements {
+		repls[repl.GetResource()] = strings.Join(repl.GetPaths(), ", ")
+	}
+
+	templateAdmins, err := c.store.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleTemplateAdmin},
+	})
+	if err != nil {
+		return xerrors.Errorf("fetch template admins: %w", err)
+	}
+
+	var notifErr error
+	for _, templateAdmin := range templateAdmins {
+		if _, err := c.notifEnq.EnqueueWithData(ctx, templateAdmin.ID, notifications.TemplateWorkspaceResourceReplaced,
+			map[string]string{
+				"org":                 org.Name,
+				"workspace":           workspace.Name,
+				"template":            workspace.TemplateName,
+				"template_version":    templateVersion.Name,
+				"preset":              prebuildPreset.Name,
+				"workspace_build_num": fmt.Sprintf("%d", build.BuildNumber),
+				"claimant":            claimant.Username,
+			},
+			map[string]any{
+				"replacements": repls,
+			}, "prebuilds_reconciler",
+			// Associate this notification with all the related entities.
+			workspace.ID, workspace.OwnerID, workspace.TemplateID, templateVersion.ID, prebuildPreset.ID, workspace.OrganizationID,
+		); err != nil {
+			notifErr = errors.Join(xerrors.Errorf("send notification to %q: %w", templateAdmin.ID.String(), err))
+			continue
+		}
+	}
+
+	return notifErr
+}

enterprise/coderd/prebuilds/reconcile_test.go

@@ -9,10 +9,14 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/coderd/notifications"
+	"github.com/coder/coder/v2/coderd/notifications/notificationstest"
 	"github.com/coder/coder/v2/coderd/util/slice"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
@@ -49,7 +53,7 @@ func TestNoReconciliationActionsIfNoPresets(t *testing.T) {
 		ReconciliationInterval: serpent.Duration(testutil.WaitLong),
 	}
 	logger := testutil.Logger(t)
-	controller := prebuilds.NewStoreReconciler(db, ps, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+	controller := prebuilds.NewStoreReconciler(db, ps, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 	// given a template version with no presets
 	org := dbgen.Organization(t, db, database.Organization{})
@@ -94,7 +98,7 @@ func TestNoReconciliationActionsIfNoPrebuilds(t *testing.T) {
 		ReconciliationInterval: serpent.Duration(testutil.WaitLong),
 	}
 	logger := testutil.Logger(t)
-	controller := prebuilds.NewStoreReconciler(db, ps, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+	controller := prebuilds.NewStoreReconciler(db, ps, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 	// given there are presets, but no prebuilds
 	org := dbgen.Organization(t, db, database.Organization{})
@@ -345,7 +349,7 @@ func TestPrebuildReconciliation(t *testing.T) {
 									1,
 									uuid.New().String(),
 								)
-								prebuild := setupTestDBPrebuild(
+								prebuild, _ := setupTestDBPrebuild(
 									t,
 									clock,
 									db,
@@ -367,7 +371,7 @@ func TestPrebuildReconciliation(t *testing.T) {
 								if useBrokenPubsub {
 									pubSub = &brokenPublisher{Pubsub: pubSub}
 								}
-								controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+								controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 								// Run the reconciliation multiple times to ensure idempotency
 								// 8 was arbitrary, but large enough to reasonably trust the result
@@ -444,7 +448,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -477,7 +481,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 	)
 	prebuildIDs := make([]uuid.UUID, 0)
 	for i := 0; i < int(preset.DesiredInstances.Int32); i++ {
-		prebuild := setupTestDBPrebuild(
+		prebuild, _ := setupTestDBPrebuild(
 			t,
 			clock,
 			db,
@@ -528,7 +532,7 @@ func TestInvalidPreset(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -592,7 +596,7 @@ func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry())
+	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, quartz.NewMock(t), prometheus.NewRegistry(), newNoopEnqueuer())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -601,7 +605,7 @@ func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) {
 	org, template := setupTestDBTemplate(t, db, ownerID, templateDeleted)
 	templateVersionID := setupTestDBTemplateVersion(ctx, t, clock, db, pubSub, org.ID, ownerID, template.ID)
 	preset := setupTestDBPreset(t, db, templateVersionID, 1, uuid.New().String())
-	prebuiltWorkspace := setupTestDBPrebuild(
+	prebuiltWorkspace, _ := setupTestDBPrebuild(
 		t,
 		clock,
 		db,
@@ -669,7 +673,7 @@ func TestRunLoop(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
-	reconciler := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock, prometheus.NewRegistry())
+	reconciler := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer())
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -702,7 +706,7 @@ func TestRunLoop(t *testing.T) {
 	)
 	prebuildIDs := make([]uuid.UUID, 0)
 	for i := 0; i < int(preset.DesiredInstances.Int32); i++ {
-		prebuild := setupTestDBPrebuild(
+		prebuild, _ := setupTestDBPrebuild(
 			t,
 			clock,
 			db,
@@ -799,7 +803,7 @@ func TestFailedBuildBackoff(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, ps := dbtestutil.NewDB(t)
-	reconciler := prebuilds.NewStoreReconciler(db, ps, cfg, logger, clock, prometheus.NewRegistry())
+	reconciler := prebuilds.NewStoreReconciler(db, ps, cfg, logger, clock, prometheus.NewRegistry(), newNoopEnqueuer())
 
 	// Given: an active template version with presets and prebuilds configured.
 	const desiredInstances = 2
@@ -812,7 +816,7 @@ func TestFailedBuildBackoff(t *testing.T) {
 
 	preset := setupTestDBPreset(t, db, templateVersionID, desiredInstances, "test")
 	for range desiredInstances {
-		_ = setupTestDBPrebuild(t, clock, db, ps, database.WorkspaceTransitionStart, database.ProvisionerJobStatusFailed, org.ID, preset, template.ID, templateVersionID)
+		_, _ = setupTestDBPrebuild(t, clock, db, ps, database.WorkspaceTransitionStart, database.ProvisionerJobStatusFailed, org.ID, preset, template.ID, templateVersionID)
 	}
 
 	// When: determining what actions to take next, backoff is calculated because the prebuild is in a failed state.
@@ -873,7 +877,7 @@ func TestFailedBuildBackoff(t *testing.T) {
 		if i == 1 {
 			status = database.ProvisionerJobStatusSucceeded
 		}
-		_ = setupTestDBPrebuild(t, clock, db, ps, database.WorkspaceTransitionStart, status, org.ID, preset, template.ID, templateVersionID)
+		_, _ = setupTestDBPrebuild(t, clock, db, ps, database.WorkspaceTransitionStart, status, org.ID, preset, template.ID, templateVersionID)
 	}
 
 	// Then: the backoff time is roughly equal to two backoff intervals, since another build has failed.
@@ -914,7 +918,8 @@ func TestReconciliationLock(t *testing.T) {
 				codersdk.PrebuildsConfig{},
 				slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug),
 				quartz.NewMock(t),
-				prometheus.NewRegistry())
+				prometheus.NewRegistry(),
+				newNoopEnqueuer())
 			reconciler.WithReconciliationLock(ctx, logger, func(_ context.Context, _ database.Store) error {
 				lockObtained := mutex.TryLock()
 				// As long as the postgres lock is held, this mutex should always be unlocked when we get here.
@@ -931,6 +936,102 @@ func TestReconciliationLock(t *testing.T) {
 	wg.Wait()
 }
 
+func TestTrackResourceReplacement(t *testing.T) {
+	t.Parallel()
+
+	if !dbtestutil.WillUsePostgres() {
+		t.Skip("This test requires postgres")
+	}
+
+	ctx := testutil.Context(t, testutil.WaitSuperLong)
+
+	// Setup.
+	clock := quartz.NewMock(t)
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
+	db, ps := dbtestutil.NewDB(t)
+
+	fakeEnqueuer := newFakeEnqueuer()
+	registry := prometheus.NewRegistry()
+	reconciler := prebuilds.NewStoreReconciler(db, ps, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer)
+
+	// Given: a template admin to receive a notification.
+	templateAdmin := dbgen.User(t, db, database.User{
+		RBACRoles: []string{codersdk.RoleTemplateAdmin},
+	})
+
+	// Given: a prebuilt workspace.
+	userID := uuid.New()
+	dbgen.User(t, db, database.User{ID: userID})
+	org, template := setupTestDBTemplate(t, db, userID, false)
+	templateVersionID := setupTestDBTemplateVersion(ctx, t, clock, db, ps, org.ID, userID, template.ID)
+	preset := setupTestDBPreset(t, db, templateVersionID, 1, "b0rked")
+	prebuiltWorkspace, prebuild := setupTestDBPrebuild(t, clock, db, ps, database.WorkspaceTransitionStart, database.ProvisionerJobStatusSucceeded, org.ID, preset, template.ID, templateVersionID)
+
+	// Given: no replacement has been tracked yet, we should not see a metric for it yet.
+	require.NoError(t, reconciler.ForceMetricsUpdate(ctx))
+	mf, err := registry.Gather()
+	require.NoError(t, err)
+	require.Nil(t, findMetric(mf, prebuilds.MetricResourceReplacementsCount, map[string]string{
+		"template_name": template.Name,
+		"preset_name":   preset.Name,
+		"org_name":      org.Name,
+	}))
+
+	// When: a claim occurred and resource replacements are detected (_how_ is out of scope of this test).
+	reconciler.TrackResourceReplacement(ctx, prebuiltWorkspace.ID, prebuild.ID, []*sdkproto.ResourceReplacement{
+		{
+			Resource: "docker_container[0]",
+			Paths:    []string{"env", "image"},
+		},
+		{
+			Resource: "docker_volume[0]",
+			Paths:    []string{"name"},
+		},
+	})
+
+	// Then: a notification will be sent detailing the replacement(s).
+	matching := fakeEnqueuer.Sent(func(notification *notificationstest.FakeNotification) bool {
+		// This is not an exhaustive check of the expected labels/data in the notification. This would tie the implementations
+		// too tightly together.
+		// All we need to validate is that a template of the right kind was sent, to the expected user, with some replacements.
+
+		if !assert.Equal(t, notification.TemplateID, notifications.TemplateWorkspaceResourceReplaced, "unexpected template") {
+			return false
+		}
+
+		if !assert.Equal(t, templateAdmin.ID, notification.UserID, "unexpected receiver") {
+			return false
+		}
+
+		if !assert.Len(t, notification.Data["replacements"], 2, "unexpected replacements count") {
+			return false
+		}
+
+		return true
+	})
+	require.Len(t, matching, 1)
+
+	// Then: the metric will be incremented.
+	mf, err = registry.Gather()
+	require.NoError(t, err)
+	metric := findMetric(mf, prebuilds.MetricResourceReplacementsCount, map[string]string{
+		"template_name": template.Name,
+		"preset_name":   preset.Name,
+		"org_name":      org.Name,
+	})
+	require.NotNil(t, metric)
+	require.NotNil(t, metric.GetCounter())
+	require.EqualValues(t, 1, metric.GetCounter().GetValue())
+}
+
+func newNoopEnqueuer() *notifications.NoopEnqueuer {
+	return notifications.NewNoopEnqueuer()
+}
+
+func newFakeEnqueuer() *notificationstest.FakeEnqueuer {
+	return notificationstest.NewFakeEnqueuer()
+}
+
 // nolint:revive // It's a control flag, but this is a test.
 func setupTestDBTemplate(
 	t *testing.T,
@@ -1040,7 +1141,7 @@ func setupTestDBPrebuild(
 	preset database.TemplateVersionPreset,
 	templateID uuid.UUID,
 	templateVersionID uuid.UUID,
-) database.WorkspaceTable {
+) (database.WorkspaceTable, database.WorkspaceBuild) {
 	t.Helper()
 	return setupTestDBWorkspace(t, clock, db, ps, transition, prebuildStatus, orgID, preset, templateID, templateVersionID, agplprebuilds.SystemUserID, agplprebuilds.SystemUserID)
 }
@@ -1058,7 +1159,7 @@ func setupTestDBWorkspace(
 	templateVersionID uuid.UUID,
 	initiatorID uuid.UUID,
 	ownerID uuid.UUID,
-) database.WorkspaceTable {
+) (database.WorkspaceTable, database.WorkspaceBuild) {
 	t.Helper()
 	cancelledAt := sql.NullTime{}
 	completedAt := sql.NullTime{}
@@ -1117,7 +1218,7 @@ func setupTestDBWorkspace(
 		},
 	})
 
-	return workspace
+	return workspace, workspaceBuild
 }
 
 // nolint:revive // It's a control flag, but this is a test.

enterprise/coderd/provisionerdaemons.go

@@ -19,6 +19,8 @@ import (
 	"storj.io/drpc/drpcserver"
 
 	"cdr.dev/slog"
+	"github.com/coder/websocket"
+
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
@@ -34,7 +36,6 @@ import (
 	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/provisionerd/proto"
 	"github.com/coder/coder/v2/provisionersdk"
-	"github.com/coder/websocket"
 )
 
 func (api *API) provisionerDaemonsEnabledMW(next http.Handler) http.Handler {
@@ -357,6 +358,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 			Clock:               api.Clock,
 		},
 		api.NotificationsEnqueuer,
+		&api.AGPL.PrebuildsReconciler,
 	)
 	if err != nil {
 		if !xerrors.Is(err, context.Canceled) {

provisioner/terraform/executor.go

@@ -260,7 +260,7 @@ func getStateFilePath(workdir string) string {
 }
 
 // revive:disable-next-line:flag-parameter
-func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr logSink, destroy bool) (*proto.PlanComplete, error) {
+func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr logSink, metadata *proto.Metadata) (*proto.PlanComplete, error) {
 	ctx, span := e.server.startTrace(ctx, tracing.FuncName())
 	defer span.End()
 
@@ -276,6 +276,7 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 		"-refresh=true",
 		"-out=" + planfilePath,
 	}
+	destroy := metadata.GetWorkspaceTransition() == proto.WorkspaceTransition_DESTROY
 	if destroy {
 		args = append(args, "-destroy")
 	}
@@ -304,7 +305,11 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 	state, plan, err := e.planResources(ctx, killCtx, planfilePath)
 	if err != nil {
 		graphTimings.ingest(createGraphTimingsEvent(timingGraphErrored))
-		return nil, err
+		return nil, xerrors.Errorf("plan resources: %w", err)
+	}
+	planJSON, err := json.Marshal(plan)
+	if err != nil {
+		return nil, xerrors.Errorf("marshal plan: %w", err)
 	}
 
 	graphTimings.ingest(createGraphTimingsEvent(timingGraphComplete))
@@ -315,13 +320,40 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 		e.logger.Warn(ctx, "failed to archive terraform modules", slog.Error(err))
 	}
 
+	// When a prebuild claim attempt is made, log a warning if a resource is due to be replaced, since this will obviate
+	// the point of prebuilding if the expensive resource is replaced once claimed!
+	var (
+		isPrebuildClaimAttempt = !destroy && metadata.GetPrebuiltWorkspaceBuildStage().IsPrebuiltWorkspaceClaim()
+		resReps                []*proto.ResourceReplacement
+	)
+	if repsFromPlan := findResourceReplacements(plan); len(repsFromPlan) > 0 {
+		if isPrebuildClaimAttempt {
+			// TODO(dannyk): we should log drift always (not just during prebuild claim attempts); we're validating that this output
+			//				 will not be overwhelming for end-users, but it'll certainly be super valuable for template admins
+			//				 to diagnose this resource replacement issue, at least.
+			//				 Once prebuilds moves out of beta, consider deleting this condition.
+
+			// Lock held before calling (see top of method).
+			e.logDrift(ctx, killCtx, planfilePath, logr)
+		}
+
+		resReps = make([]*proto.ResourceReplacement, 0, len(repsFromPlan))
+		for n, p := range repsFromPlan {
+			resReps = append(resReps, &proto.ResourceReplacement{
+				Resource: n,
+				Paths:    p,
+			})
+		}
+	}
+
 	msg := &proto.PlanComplete{
 		Parameters:            state.Parameters,
 		Resources:             state.Resources,
 		ExternalAuthProviders: state.ExternalAuthProviders,
 		Timings:               append(e.timings.aggregate(), graphTimings.aggregate()...),
 		Presets:               state.Presets,
-		Plan:                  plan,
+		Plan:                  planJSON,
+		ResourceReplacements:  resReps,
 		ModuleFiles:           moduleFiles,
 	}
 
@@ -350,80 +382,16 @@ func onlyDataResources(sm tfjson.StateModule) tfjson.StateModule {
 	return filtered
 }
 
-func (e *executor) logResourceReplacements(ctx context.Context, plan *tfjson.Plan) {
-	if plan == nil {
-		return
-	}
-
-	if len(plan.ResourceChanges) == 0 {
-		return
-	}
-	var (
-		count        int
-		replacements = make(map[string][]string, len(plan.ResourceChanges))
-	)
-
-	for _, ch := range plan.ResourceChanges {
-		// No change, no problem!
-		if ch.Change == nil {
-			continue
-		}
-
-		// No-op change, no problem!
-		if ch.Change.Actions.NoOp() {
-			continue
-		}
-
-		// No replacements, no problem!
-		if len(ch.Change.ReplacePaths) == 0 {
-			continue
-		}
-
-		// Replacing our resources, no problem!
-		if strings.Index(ch.Type, "coder_") == 0 {
-			continue
-		}
-
-		for _, p := range ch.Change.ReplacePaths {
-			var path string
-			switch p := p.(type) {
-			case []interface{}:
-				segs := p
-				list := make([]string, 0, len(segs))
-				for _, s := range segs {
-					list = append(list, fmt.Sprintf("%v", s))
-				}
-				path = strings.Join(list, ".")
-			default:
-				path = fmt.Sprintf("%v", p)
-			}
-
-			replacements[ch.Address] = append(replacements[ch.Address], path)
-		}
-
-		count++
-	}
-
-	if count > 0 {
-		e.server.logger.Warn(ctx, "plan introduces resource changes", slog.F("count", count))
-		for n, p := range replacements {
-			e.server.logger.Warn(ctx, "resource will be replaced", slog.F("name", n), slog.F("replacement_paths", strings.Join(p, ",")))
-		}
-	}
-}
-
 // planResources must only be called while the lock is held.
-func (e *executor) planResources(ctx, killCtx context.Context, planfilePath string) (*State, json.RawMessage, error) {
+func (e *executor) planResources(ctx, killCtx context.Context, planfilePath string) (*State, *tfjson.Plan, error) {
 	ctx, span := e.server.startTrace(ctx, tracing.FuncName())
 	defer span.End()
 
-	plan, err := e.showPlan(ctx, killCtx, planfilePath)
+	plan, err := e.parsePlan(ctx, killCtx, planfilePath)
 	if err != nil {
 		return nil, nil, xerrors.Errorf("show terraform plan file: %w", err)
 	}
 
-	e.logResourceReplacements(ctx, plan)
-
 	rawGraph, err := e.graph(ctx, killCtx)
 	if err != nil {
 		return nil, nil, xerrors.Errorf("graph: %w", err)
@@ -447,16 +415,11 @@ func (e *executor) planResources(ctx, killCtx context.Context, planfilePath stri
 		return nil, nil, err
 	}
 
-	planJSON, err := json.Marshal(plan)
+	return state, plan, nil
-	if err != nil {
-		return nil, nil, err
-	}
-
-	return state, planJSON, nil
 }
 
-// showPlan must only be called while the lock is held.
+// parsePlan must only be called while the lock is held.
-func (e *executor) showPlan(ctx, killCtx context.Context, planfilePath string) (*tfjson.Plan, error) {
+func (e *executor) parsePlan(ctx, killCtx context.Context, planfilePath string) (*tfjson.Plan, error) {
 	ctx, span := e.server.startTrace(ctx, tracing.FuncName())
 	defer span.End()
 
@@ -466,6 +429,64 @@ func (e *executor) showPlan(ctx, killCtx context.Context, planfilePath string) (
 	return p, err
 }
 
+// logDrift must only be called while the lock is held.
+// It will log the output of `terraform show`, which will show which resources have drifted from the known state.
+func (e *executor) logDrift(ctx, killCtx context.Context, planfilePath string, logr logSink) {
+	stdout, stdoutDone := resourceReplaceLogWriter(logr, e.logger)
+	stderr, stderrDone := logWriter(logr, proto.LogLevel_ERROR)
+	defer func() {
+		_ = stdout.Close()
+		_ = stderr.Close()
+		<-stdoutDone
+		<-stderrDone
+	}()
+
+	err := e.showPlan(ctx, killCtx, stdout, stderr, planfilePath)
+	if err != nil {
+		e.server.logger.Debug(ctx, "failed to log state drift", slog.Error(err))
+	}
+}
+
+// resourceReplaceLogWriter highlights log lines relating to resource replacement by elevating their log level.
+// This will help template admins to visually find problematic resources easier.
+//
+// The WriteCloser must be closed by the caller to end logging, after which the returned channel will be closed to
+// indicate that logging of the written data has finished.  Failure to close the WriteCloser will leak a goroutine.
+func resourceReplaceLogWriter(sink logSink, logger slog.Logger) (io.WriteCloser, <-chan struct{}) {
+	r, w := io.Pipe()
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+
+		scanner := bufio.NewScanner(r)
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			level := proto.LogLevel_INFO
+
+			// Terraform indicates that a resource will be deleted and recreated by showing the change along with this substring.
+			if bytes.Contains(line, []byte("# forces replacement")) {
+				level = proto.LogLevel_WARN
+			}
+
+			sink.ProvisionLog(level, string(line))
+		}
+		if err := scanner.Err(); err != nil {
+			logger.Error(context.Background(), "failed to read terraform log", slog.Error(err))
+		}
+	}()
+	return w, done
+}
+
+// showPlan must only be called while the lock is held.
+func (e *executor) showPlan(ctx, killCtx context.Context, stdoutWriter, stderrWriter io.WriteCloser, planfilePath string) error {
+	ctx, span := e.server.startTrace(ctx, tracing.FuncName())
+	defer span.End()
+
+	args := []string{"show", "-no-color", planfilePath}
+	return e.execWriteOutput(ctx, killCtx, args, e.basicEnv(), stdoutWriter, stderrWriter)
+}
+
 // graph must only be called while the lock is held.
 func (e *executor) graph(ctx, killCtx context.Context) (string, error) {
 	ctx, span := e.server.startTrace(ctx, tracing.FuncName())

provisioner/terraform/provision.go

@@ -163,10 +163,7 @@ func (s *server) Plan(
 		return provisionersdk.PlanErrorf("plan vars: %s", err)
 	}
 
-	resp, err := e.plan(
+	resp, err := e.plan(ctx, killCtx, env, vars, sess, request.Metadata)
-		ctx, killCtx, env, vars, sess,
-		request.Metadata.GetWorkspaceTransition() == proto.WorkspaceTransition_DESTROY,
-	)
 	if err != nil {
 		return provisionersdk.PlanErrorf("%s", err.Error())
 	}

provisioner/terraform/resource_replacements.go

@@ -0,0 +1,86 @@
+package terraform
+
+import (
+	"fmt"
+	"strings"
+
+	tfjson "github.com/hashicorp/terraform-json"
+)
+
+type resourceReplacements map[string][]string
+
+// resourceReplacements finds all resources which would be replaced by the current plan, and the attribute paths which
+// caused the replacement.
+//
+// NOTE: "replacement" in terraform terms means that a resource will have to be destroyed and replaced with a new resource
+// since one of its immutable attributes was modified, which cannot be updated in-place.
+func findResourceReplacements(plan *tfjson.Plan) resourceReplacements {
+	if plan == nil {
+		return nil
+	}
+
+	// No changes, no problem!
+	if len(plan.ResourceChanges) == 0 {
+		return nil
+	}
+
+	replacements := make(resourceReplacements, len(plan.ResourceChanges))
+
+	for _, ch := range plan.ResourceChanges {
+		// No change, no problem!
+		if ch.Change == nil {
+			continue
+		}
+
+		// No-op change, no problem!
+		if ch.Change.Actions.NoOp() {
+			continue
+		}
+
+		// No replacements, no problem!
+		if len(ch.Change.ReplacePaths) == 0 {
+			continue
+		}
+
+		// Replacing our resources: could be a problem - but we ignore since they're "virtual" resources. If any of these
+		// resources' attributes are referenced by non-coder resources, those will show up as transitive changes there.
+		// i.e. if the coder_agent.id attribute is used in docker_container.env
+		//
+		// Replacing our resources is not strictly a problem in and of itself.
+		//
+		// NOTE:
+		// We may need to special-case coder_agent in the future. Currently, coder_agent is replaced on every build
+		// because it only supports Create but not Update: https://github.com/coder/terraform-provider-coder/blob/5648efb/provider/agent.go#L28
+		// When we can modify an agent's attributes, some of which may be immutable (like "arch") and some may not (like "env"),
+		// then we'll have to handle this specifically.
+		// This will only become relevant once we support multiple agents: https://github.com/coder/coder/issues/17388
+		if strings.Index(ch.Type, "coder_") == 0 {
+			continue
+		}
+
+		// Replacements found, problem!
+		for _, val := range ch.Change.ReplacePaths {
+			var pathStr string
+			// Each path needs to be coerced into a string. All types except []interface{} can be coerced using fmt.Sprintf.
+			switch path := val.(type) {
+			case []interface{}:
+				// Found a slice of paths; coerce to string and join by ".".
+				segments := make([]string, 0, len(path))
+				for _, seg := range path {
+					segments = append(segments, fmt.Sprintf("%v", seg))
+				}
+				pathStr = strings.Join(segments, ".")
+			default:
+				pathStr = fmt.Sprintf("%v", path)
+			}
+
+			replacements[ch.Address] = append(replacements[ch.Address], pathStr)
+		}
+	}
+
+	if len(replacements) == 0 {
+		return nil
+	}
+
+	return replacements
+}

provisioner/terraform/resource_replacements_internal_test.go

@@ -0,0 +1,176 @@
+package terraform
+
+import (
+	"testing"
+
+	tfjson "github.com/hashicorp/terraform-json"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindResourceReplacements(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name     string
+		plan     *tfjson.Plan
+		expected resourceReplacements
+	}{
+		{
+			name: "nil plan",
+		},
+		{
+			name: "no resource changes",
+			plan: &tfjson.Plan{},
+		},
+		{
+			name: "resource change with nil change",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+					},
+				},
+			},
+		},
+		{
+			name: "no-op action",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Change: &tfjson.Change{
+							Actions: tfjson.Actions{tfjson.ActionNoop},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "empty replace paths",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Change: &tfjson.Change{
+							Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "coder_* types are ignored",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Type:    "coder_resource",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"path1"},
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "valid replacements - single path",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Type:    "example_resource",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"path1"},
+						},
+					},
+				},
+			},
+			expected: resourceReplacements{
+				"resource1": {"path1"},
+			},
+		},
+		{
+			name: "valid replacements - multiple paths",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Type:    "example_resource",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"path1", "path2"},
+						},
+					},
+				},
+			},
+			expected: resourceReplacements{
+				"resource1": {"path1", "path2"},
+			},
+		},
+		{
+			name: "complex replace path",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Type:    "example_resource",
+						Change: &tfjson.Change{
+							Actions: tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{
+								[]interface{}{"path", "to", "key"},
+							},
+						},
+					},
+				},
+			},
+			expected: resourceReplacements{
+				"resource1": {"path.to.key"},
+			},
+		},
+		{
+			name: "multiple changes",
+			plan: &tfjson.Plan{
+				ResourceChanges: []*tfjson.ResourceChange{
+					{
+						Address: "resource1",
+						Type:    "example_resource",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"path1"},
+						},
+					},
+					{
+						Address: "resource2",
+						Type:    "example_resource",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"path2", "path3"},
+						},
+					},
+					{
+						Address: "resource3",
+						Type:    "coder_example",
+						Change: &tfjson.Change{
+							Actions:      tfjson.Actions{tfjson.ActionDelete, tfjson.ActionCreate},
+							ReplacePaths: []interface{}{"ignored_path"},
+						},
+					},
+				},
+			},
+			expected: resourceReplacements{
+				"resource1": {"path1"},
+				"resource2": {"path2", "path3"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			require.EqualValues(t, tc.expected, findResourceReplacements(tc.plan))
+		})
+	}
+}

provisionerd/proto/provisionerd.pb.go

@@ -1223,10 +1223,11 @@ type CompletedJob_WorkspaceBuild struct {
 	sizeCache     protoimpl.SizeCache
 	unknownFields protoimpl.UnknownFields
 
-	State     []byte            `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
+	State                []byte                       `protobuf:"bytes,1,opt,name=state,proto3" json:"state,omitempty"`
-	Resources []*proto.Resource `protobuf:"bytes,2,rep,name=resources,proto3" json:"resources,omitempty"`
+	Resources            []*proto.Resource            `protobuf:"bytes,2,rep,name=resources,proto3" json:"resources,omitempty"`
-	Timings   []*proto.Timing   `protobuf:"bytes,3,rep,name=timings,proto3" json:"timings,omitempty"`
+	Timings              []*proto.Timing              `protobuf:"bytes,3,rep,name=timings,proto3" json:"timings,omitempty"`
-	Modules   []*proto.Module   `protobuf:"bytes,4,rep,name=modules,proto3" json:"modules,omitempty"`
+	Modules              []*proto.Module              `protobuf:"bytes,4,rep,name=modules,proto3" json:"modules,omitempty"`
+	ResourceReplacements []*proto.ResourceReplacement `protobuf:"bytes,5,rep,name=resource_replacements,json=resourceReplacements,proto3" json:"resource_replacements,omitempty"`
 }
 
 func (x *CompletedJob_WorkspaceBuild) Reset() {
@@ -1289,6 +1290,13 @@ func (x *CompletedJob_WorkspaceBuild) GetModules() []*proto.Module {
 	return nil
 }
 
+func (x *CompletedJob_WorkspaceBuild) GetResourceReplacements() []*proto.ResourceReplacement {
+	if x != nil {
+		return x.ResourceReplacements
+	}
+	return nil
+}
+
 type CompletedJob_TemplateImport struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -1597,7 +1605,7 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{
 	0x6d, 0x69, 0x6e, 0x67, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x69, 0x6e, 0x67, 0x73, 0x1a, 0x10, 0x0a,
 	0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x1a,
 	0x10, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x44, 0x72, 0x79, 0x52, 0x75,
-	0x6e, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xb6, 0x09, 0x0a, 0x0c, 0x43, 0x6f,
+	0x6e, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x8d, 0x0a, 0x0a, 0x0c, 0x43, 0x6f,
 	0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f,
 	0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49,
 	0x64, 0x12, 0x54, 0x0a, 0x0f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x62,
@@ -1616,7 +1624,7 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{
 	0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64,
 	0x4a, 0x6f, 0x62, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x44, 0x72, 0x79, 0x52,
 	0x75, 0x6e, 0x48, 0x00, 0x52, 0x0e, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x44, 0x72,
-	0x79, 0x52, 0x75, 0x6e, 0x1a, 0xb9, 0x01, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61,
+	0x79, 0x52, 0x75, 0x6e, 0x1a, 0x90, 0x02, 0x0a, 0x0e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61,
 	0x63, 0x65, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65,
 	0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x33, 0x0a,
 	0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
@@ -1628,145 +1636,150 @@ var file_provisionerd_proto_provisionerd_proto_rawDesc = []byte{
 	0x73, 0x12, 0x2d, 0x0a, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03,
 	0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
 	0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73,
-	0x1a, 0xd1, 0x04, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x49, 0x6d, 0x70,
+	0x12, 0x55, 0x0a, 0x15, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x70,
-	0x6f, 0x72, 0x74, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73,
+	0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70,
+	0x20, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65,
-	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75,
+	0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65, 0x6e,
-	0x72, 0x63, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72,
+	0x74, 0x52, 0x14, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x61,
-	0x63, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x6f,
+	0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x1a, 0xd1, 0x04, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70,
-	0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72,
+	0x6c, 0x61, 0x74, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x3e, 0x0a, 0x0f, 0x73, 0x74,
-	0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72,
+	0x61, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20,
-	0x63, 0x65, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65,
-	0x73, 0x12, 0x43, 0x0a, 0x0f, 0x72, 0x69, 0x63, 0x68, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65,
+	0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0e, 0x73, 0x74, 0x61, 0x72,
-	0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x0e, 0x73, 0x74,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x69, 0x63, 0x68, 0x50, 0x61, 0x72,
+	0x6f, 0x70, 0x5f, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03,
-	0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x0e, 0x72, 0x69, 0x63, 0x68, 0x50, 0x61, 0x72, 0x61,
+	0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
-	0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x41, 0x0a, 0x1d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e,
+	0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, 0x73, 0x74, 0x6f, 0x70, 0x52,
-	0x61, 0x6c, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72,
+	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0f, 0x72, 0x69, 0x63, 0x68,
-	0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x1a, 0x65,
+	0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28,
-	0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x75, 0x74, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69,
+	0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e,
-	0x64, 0x65, 0x72, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x61, 0x0a, 0x17, 0x65, 0x78, 0x74,
+	0x52, 0x69, 0x63, 0x68, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x52, 0x0e, 0x72,
-	0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69,
+	0x69, 0x63, 0x68, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x41, 0x0a,
-	0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x70, 0x72, 0x6f,
+	0x1d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x5f, 0x70,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61,
+	0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04,
-	0x6c, 0x41, 0x75, 0x74, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73,
+	0x20, 0x03, 0x28, 0x09, 0x52, 0x1a, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x75,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41,
+	0x74, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x4e, 0x61, 0x6d, 0x65, 0x73,
-	0x75, 0x74, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x12, 0x38, 0x0a, 0x0d,
+	0x12, 0x61, 0x0a, 0x17, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x75, 0x74,
-	0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x06, 0x20,
+	0x68, 0x5f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28,
+	0x0b, 0x32, 0x29, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e,
+	0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x75, 0x74, 0x68, 0x50, 0x72, 0x6f, 0x76,
+	0x69, 0x64, 0x65, 0x72, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x15, 0x65, 0x78,
+	0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x75, 0x74, 0x68, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64,
+	0x65, 0x72, 0x73, 0x12, 0x38, 0x0a, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x6f, 0x64,
+	0x75, 0x6c, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f,
+	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52,
+	0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x36, 0x0a,
+	0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20,
 	0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65,
-	0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4d,
+	0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x70, 0x4d, 0x6f,
-	0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x70, 0x5f, 0x6d,
+	0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x74, 0x73,
-	0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70,
+	0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69,
-	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c,
+	0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x52, 0x07, 0x70, 0x72, 0x65,
-	0x65, 0x52, 0x0b, 0x73, 0x74, 0x6f, 0x70, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x12, 0x2d,
+	0x73, 0x65, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6c, 0x61, 0x6e, 0x18, 0x09, 0x20, 0x01,
-	0x0a, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x74, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x28, 0x0c, 0x52, 0x04, 0x70, 0x6c, 0x61, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x75,
-	0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x50, 0x72,
+	0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b,
-	0x65, 0x73, 0x65, 0x74, 0x52, 0x07, 0x70, 0x72, 0x65, 0x73, 0x65, 0x74, 0x73, 0x12, 0x12, 0x0a,
+	0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x1a, 0x74, 0x0a, 0x0e, 0x54,
-	0x04, 0x70, 0x6c, 0x61, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x70, 0x6c, 0x61,
+	0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x44, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x12, 0x33, 0x0a,
-	0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65,
+	0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b,
-	0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x46,
+	0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52,
-	0x69, 0x6c, 0x65, 0x73, 0x1a, 0x74, 0x0a, 0x0e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65,
+	0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63,
-	0x44, 0x72, 0x79, 0x52, 0x75, 0x6e, 0x12, 0x33, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72,
+	0x65, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20,
-	0x63, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76,
+	0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65,
-	0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65,
+	0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65,
-	0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x6d,
+	0x73, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x03, 0x4c, 0x6f,
-	0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x70,
+	0x67, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x4d, 0x6f, 0x64, 0x75, 0x6c,
+	0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
-	0x65, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x74, 0x79,
+	0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72,
-	0x70, 0x65, 0x22, 0xb0, 0x01, 0x0a, 0x03, 0x4c, 0x6f, 0x67, 0x12, 0x2f, 0x0a, 0x06, 0x73, 0x6f,
+	0x63, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x70, 0x72, 0x6f,
+	0x0e, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75,
+	0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12,
-	0x72, 0x63, 0x65, 0x52, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x2b, 0x0a, 0x05, 0x6c,
+	0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20,
-	0x65, 0x76, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x70, 0x72, 0x6f,
+	0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x14,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x4c, 0x6f, 0x67, 0x4c, 0x65, 0x76, 0x65,
+	0x0a, 0x05, 0x73, 0x74, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73,
-	0x6c, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61,
+	0x74, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x05,
-	0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x63, 0x72,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0xa6, 0x03, 0x0a,
-	0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x67, 0x65,
+	0x10, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x67, 0x65, 0x12, 0x16, 0x0a,
+	0x74, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
-	0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f,
+	0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73,
-	0x75, 0x74, 0x70, 0x75, 0x74, 0x22, 0xa6, 0x03, 0x0a, 0x10, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65,
+	0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69,
-	0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f,
+	0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c, 0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12,
-	0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49,
+	0x4c, 0x0a, 0x12, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x5f, 0x76, 0x61, 0x72, 0x69,
-	0x64, 0x12, 0x25, 0x0a, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32,
+	0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x70, 0x72,
-	0x11, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x4c,
+	0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61,
-	0x6f, 0x67, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x4c, 0x0a, 0x12, 0x74, 0x65, 0x6d, 0x70,
+	0x74, 0x65, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x11, 0x74, 0x65, 0x6d, 0x70,
-	0x6c, 0x61, 0x74, 0x65, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x04,
+	0x6c, 0x61, 0x74, 0x65, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x4c, 0x0a,
-	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e,
+	0x14, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x76,
-	0x65, 0x72, 0x2e, 0x54, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x61, 0x72, 0x69, 0x61,
+	0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72,
-	0x62, 0x6c, 0x65, 0x52, 0x11, 0x74, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x74, 0x65, 0x56, 0x61, 0x72,
+	0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62,
-	0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x4c, 0x0a, 0x14, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x76,
+	0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x56, 0x61, 0x72,
-	0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x05,
+	0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x72,
-	0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e,
+	0x65, 0x61, 0x64, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x72, 0x65, 0x61,
-	0x65, 0x72, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65,
+	0x64, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65,
-	0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61,
+	0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x70, 0x72,
-	0x6c, 0x75, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x64, 0x6d, 0x65, 0x18, 0x06,
+	0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74,
-	0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x72, 0x65, 0x61, 0x64, 0x6d, 0x65, 0x12, 0x58, 0x0a, 0x0e,
+	0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x6f, 0x72, 0x6b,
-	0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x74, 0x61, 0x67, 0x73, 0x18, 0x07,
+	0x73, 0x70, 0x61, 0x63, 0x65, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d,
-	0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e,
+	0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x54, 0x61, 0x67, 0x73, 0x1a, 0x40, 0x0a,
-	0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71,
+	0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e,
-	0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x54, 0x61,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
-	0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
-	0x63, 0x65, 0x54, 0x61, 0x67, 0x73, 0x1a, 0x40, 0x0a, 0x12, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a,
-	0x61, 0x63, 0x65, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03,
+	0x04, 0x08, 0x03, 0x10, 0x04, 0x22, 0x7a, 0x0a, 0x11, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a,
-	0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14,
+	0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61,
-	0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76,
+	0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x63, 0x61,
-	0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x03, 0x10, 0x04, 0x22, 0x7a,
+	0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, 0x12, 0x43, 0x0a, 0x0f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62,
-	0x0a, 0x11, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f,
+	0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32,
-	0x6e, 0x73, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, 0x18,
+	0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x56, 0x61,
-	0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x65, 0x64, 0x12,
+	0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x76, 0x61, 0x72,
-	0x43, 0x0a, 0x0f, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75,
+	0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x4a, 0x04, 0x08, 0x02, 0x10,
-	0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69,
+	0x03, 0x22, 0x4a, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61,
-	0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x56,
+	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69,
-	0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61,
+	0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x1d,
-	0x6c, 0x75, 0x65, 0x73, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x4a, 0x0a, 0x12, 0x43, 0x6f,
+	0x0a, 0x0a, 0x64, 0x61, 0x69, 0x6c, 0x79, 0x5f, 0x63, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01,
-	0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
+	0x28, 0x05, 0x52, 0x09, 0x64, 0x61, 0x69, 0x6c, 0x79, 0x43, 0x6f, 0x73, 0x74, 0x22, 0x68, 0x0a,
-	0x12, 0x15, 0x0a, 0x06, 0x6a, 0x6f, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70,
-	0x52, 0x05, 0x6a, 0x6f, 0x62, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x69, 0x6c, 0x79,
+	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08,
-	0x5f, 0x63, 0x6f, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x64, 0x61, 0x69,
+	0x52, 0x02, 0x6f, 0x6b, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x73, 0x5f,
-	0x6c, 0x79, 0x43, 0x6f, 0x73, 0x74, 0x22, 0x68, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
+	0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f,
-	0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a,
+	0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x12,
-	0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x29, 0x0a,
+	0x16, 0x0a, 0x06, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52,
-	0x10, 0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x73, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
+	0x06, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74, 0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65,
-	0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x63, 0x72, 0x65, 0x64, 0x69, 0x74, 0x73,
+	0x6c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x2a, 0x34, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x53,
-	0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x64, 0x67,
+	0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x12, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49,
-	0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x62, 0x75, 0x64, 0x67, 0x65, 0x74,
+	0x4f, 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x41, 0x45, 0x4d, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a,
-	0x22, 0x0f, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72,
+	0x0b, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x10, 0x01, 0x32, 0xc5,
-	0x65, 0x2a, 0x34, 0x0a, 0x09, 0x4c, 0x6f, 0x67, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x16,
+	0x03, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x44, 0x61,
-	0x0a, 0x12, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53, 0x49, 0x4f, 0x4e, 0x45, 0x52, 0x5f, 0x44, 0x41,
+	0x65, 0x6d, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a,
-	0x45, 0x4d, 0x4f, 0x4e, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x4f, 0x56, 0x49, 0x53,
+	0x6f, 0x62, 0x12, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72,
-	0x49, 0x4f, 0x4e, 0x45, 0x52, 0x10, 0x01, 0x32, 0xc5, 0x03, 0x0a, 0x11, 0x50, 0x72, 0x6f, 0x76,
+	0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73,
-	0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x44, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x12, 0x41, 0x0a,
+	0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a,
-	0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x13, 0x2e, 0x70, 0x72,
+	0x6f, 0x62, 0x22, 0x03, 0x88, 0x02, 0x01, 0x12, 0x52, 0x0a, 0x14, 0x41, 0x63, 0x71, 0x75, 0x69,
-	0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79,
+	0x72, 0x65, 0x4a, 0x6f, 0x62, 0x57, 0x69, 0x74, 0x68, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12,
-	0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e,
+	0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43,
-	0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x22, 0x03, 0x88, 0x02, 0x01,
+	0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x1a, 0x19, 0x2e, 0x70,
-	0x12, 0x52, 0x0a, 0x14, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x4a, 0x6f, 0x62, 0x57, 0x69,
+	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75,
-	0x74, 0x68, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x12, 0x1b, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69,
+	0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x28, 0x01, 0x30, 0x01, 0x12, 0x52, 0x0a, 0x0b, 0x43,
-	0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x41, 0x63,
+	0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f,
-	0x71, 0x75, 0x69, 0x72, 0x65, 0x1a, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
+	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
-	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x64, 0x4a, 0x6f, 0x62,
+	0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x70,
-	0x28, 0x01, 0x30, 0x01, 0x12, 0x52, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75,
+	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x6d,
-	0x6f, 0x74, 0x61, 0x12, 0x20, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65,
+	0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12,
-	0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x52, 0x65,
+	0x4c, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x70,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
+	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61,
-	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61,
+	0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70,
-	0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61,
+	0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61,
-	0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1e, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
+	0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a,
-	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65,
+	0x07, 0x46, 0x61, 0x69, 0x6c, 0x4a, 0x6f, 0x62, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69,
-	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
+	0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4a, 0x6f,
-	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x52, 0x65,
+	0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x46, 0x61, 0x69, 0x6c, 0x4a, 0x6f,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65,
-	0x62, 0x12, 0x17, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
+	0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f,
-	0x2e, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f,
+	0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12,
+	0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
-	0x3e, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4a, 0x6f, 0x62, 0x12, 0x1a,
+	0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42, 0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
-	0x2e, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x43, 0x6f,
+	0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72,
-	0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x6f, 0x62, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f,
+	0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x42,
+	0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-	0x2e, 0x5a, 0x2c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f,
-	0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f,
-	0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
-	0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -1815,9 +1828,10 @@ var file_provisionerd_proto_provisionerd_proto_goTypes = []interface{}{
 	(*proto.Timing)(nil),                       // 28: provisioner.Timing
 	(*proto.Resource)(nil),                     // 29: provisioner.Resource
 	(*proto.Module)(nil),                       // 30: provisioner.Module
-	(*proto.RichParameter)(nil),                // 31: provisioner.RichParameter
+	(*proto.ResourceReplacement)(nil),          // 31: provisioner.ResourceReplacement
-	(*proto.ExternalAuthProviderResource)(nil), // 32: provisioner.ExternalAuthProviderResource
+	(*proto.RichParameter)(nil),                // 32: provisioner.RichParameter
-	(*proto.Preset)(nil),                       // 33: provisioner.Preset
+	(*proto.ExternalAuthProviderResource)(nil), // 33: provisioner.ExternalAuthProviderResource
+	(*proto.Preset)(nil),                       // 34: provisioner.Preset
 }
 var file_provisionerd_proto_provisionerd_proto_depIdxs = []int32{
 	11, // 0: provisionerd.AcquiredJob.workspace_build:type_name -> provisionerd.AcquiredJob.WorkspaceBuild
@@ -1851,32 +1865,33 @@ var file_provisionerd_proto_provisionerd_proto_depIdxs = []int32{
 	29, // 28: provisionerd.CompletedJob.WorkspaceBuild.resources:type_name -> provisioner.Resource
 	28, // 29: provisionerd.CompletedJob.WorkspaceBuild.timings:type_name -> provisioner.Timing
 	30, // 30: provisionerd.CompletedJob.WorkspaceBuild.modules:type_name -> provisioner.Module
-	29, // 31: provisionerd.CompletedJob.TemplateImport.start_resources:type_name -> provisioner.Resource
+	31, // 31: provisionerd.CompletedJob.WorkspaceBuild.resource_replacements:type_name -> provisioner.ResourceReplacement
-	29, // 32: provisionerd.CompletedJob.TemplateImport.stop_resources:type_name -> provisioner.Resource
+	29, // 32: provisionerd.CompletedJob.TemplateImport.start_resources:type_name -> provisioner.Resource
-	31, // 33: provisionerd.CompletedJob.TemplateImport.rich_parameters:type_name -> provisioner.RichParameter
+	29, // 33: provisionerd.CompletedJob.TemplateImport.stop_resources:type_name -> provisioner.Resource
-	32, // 34: provisionerd.CompletedJob.TemplateImport.external_auth_providers:type_name -> provisioner.ExternalAuthProviderResource
+	32, // 34: provisionerd.CompletedJob.TemplateImport.rich_parameters:type_name -> provisioner.RichParameter
-	30, // 35: provisionerd.CompletedJob.TemplateImport.start_modules:type_name -> provisioner.Module
+	33, // 35: provisionerd.CompletedJob.TemplateImport.external_auth_providers:type_name -> provisioner.ExternalAuthProviderResource
-	30, // 36: provisionerd.CompletedJob.TemplateImport.stop_modules:type_name -> provisioner.Module
+	30, // 36: provisionerd.CompletedJob.TemplateImport.start_modules:type_name -> provisioner.Module
-	33, // 37: provisionerd.CompletedJob.TemplateImport.presets:type_name -> provisioner.Preset
+	30, // 37: provisionerd.CompletedJob.TemplateImport.stop_modules:type_name -> provisioner.Module
-	29, // 38: provisionerd.CompletedJob.TemplateDryRun.resources:type_name -> provisioner.Resource
+	34, // 38: provisionerd.CompletedJob.TemplateImport.presets:type_name -> provisioner.Preset
-	30, // 39: provisionerd.CompletedJob.TemplateDryRun.modules:type_name -> provisioner.Module
+	29, // 39: provisionerd.CompletedJob.TemplateDryRun.resources:type_name -> provisioner.Resource
-	1,  // 40: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty
+	30, // 40: provisionerd.CompletedJob.TemplateDryRun.modules:type_name -> provisioner.Module
-	10, // 41: provisionerd.ProvisionerDaemon.AcquireJobWithCancel:input_type -> provisionerd.CancelAcquire
+	1,  // 41: provisionerd.ProvisionerDaemon.AcquireJob:input_type -> provisionerd.Empty
-	8,  // 42: provisionerd.ProvisionerDaemon.CommitQuota:input_type -> provisionerd.CommitQuotaRequest
+	10, // 42: provisionerd.ProvisionerDaemon.AcquireJobWithCancel:input_type -> provisionerd.CancelAcquire
-	6,  // 43: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.UpdateJobRequest
+	8,  // 43: provisionerd.ProvisionerDaemon.CommitQuota:input_type -> provisionerd.CommitQuotaRequest
-	3,  // 44: provisionerd.ProvisionerDaemon.FailJob:input_type -> provisionerd.FailedJob
+	6,  // 44: provisionerd.ProvisionerDaemon.UpdateJob:input_type -> provisionerd.UpdateJobRequest
-	4,  // 45: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob
+	3,  // 45: provisionerd.ProvisionerDaemon.FailJob:input_type -> provisionerd.FailedJob
-	2,  // 46: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob
+	4,  // 46: provisionerd.ProvisionerDaemon.CompleteJob:input_type -> provisionerd.CompletedJob
-	2,  // 47: provisionerd.ProvisionerDaemon.AcquireJobWithCancel:output_type -> provisionerd.AcquiredJob
+	2,  // 47: provisionerd.ProvisionerDaemon.AcquireJob:output_type -> provisionerd.AcquiredJob
-	9,  // 48: provisionerd.ProvisionerDaemon.CommitQuota:output_type -> provisionerd.CommitQuotaResponse
+	2,  // 48: provisionerd.ProvisionerDaemon.AcquireJobWithCancel:output_type -> provisionerd.AcquiredJob
-	7,  // 49: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.UpdateJobResponse
+	9,  // 49: provisionerd.ProvisionerDaemon.CommitQuota:output_type -> provisionerd.CommitQuotaResponse
-	1,  // 50: provisionerd.ProvisionerDaemon.FailJob:output_type -> provisionerd.Empty
+	7,  // 50: provisionerd.ProvisionerDaemon.UpdateJob:output_type -> provisionerd.UpdateJobResponse
-	1,  // 51: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty
+	1,  // 51: provisionerd.ProvisionerDaemon.FailJob:output_type -> provisionerd.Empty
-	46, // [46:52] is the sub-list for method output_type
+	1,  // 52: provisionerd.ProvisionerDaemon.CompleteJob:output_type -> provisionerd.Empty
-	40, // [40:46] is the sub-list for method input_type
+	47, // [47:53] is the sub-list for method output_type
-	40, // [40:40] is the sub-list for extension type_name
+	41, // [41:47] is the sub-list for method input_type
-	40, // [40:40] is the sub-list for extension extendee
+	41, // [41:41] is the sub-list for extension type_name
-	0,  // [0:40] is the sub-list for field type_name
+	41, // [41:41] is the sub-list for extension extendee
+	0,  // [0:41] is the sub-list for field type_name
 }
 
 func init() { file_provisionerd_proto_provisionerd_proto_init() }

provisionerd/proto/provisionerd.proto

@@ -79,6 +79,7 @@ message CompletedJob {
         repeated provisioner.Resource resources = 2;
         repeated provisioner.Timing timings = 3;
         repeated provisioner.Module modules = 4;
+        repeated provisioner.ResourceReplacement resource_replacements = 5;
     }
     message TemplateImport {
         repeated provisioner.Resource start_resources = 1;

provisionerd/proto/version.go

@@ -20,6 +20,7 @@ import "github.com/coder/coder/v2/apiversion"
 //     the previous values for the `terraform apply` to enforce monotonicity
 //     in the terraform provider.
 //   - Add new field named `running_agent_auth_tokens` to provisioner job metadata
+//   - Add new field named `resource_replacements` in PlanComplete & CompletedJob.WorkspaceBuild.
 const (
 	CurrentMajor = 1
 	CurrentMinor = 5

provisionerd/runner/runner.go

@@ -1065,6 +1065,8 @@ func (r *Runner) runWorkspaceBuild(ctx context.Context) (*proto.CompletedJob, *p
 				// called by `plan`. `apply` does not modify them, so we can use the
 				// modules from the plan response.
 				Modules: planComplete.Modules,
+				// Resource replacements are discovered at plan time, only.
+				ResourceReplacements: planComplete.ResourceReplacements,
 			},
 		},
 	}, nil

provisionersdk/proto/provisioner.proto

@@ -73,6 +73,11 @@ message PresetParameter {
     string value = 2;
 }
 
+message ResourceReplacement {
+    string resource = 1;
+    repeated string paths = 2;
+}
+
 // VariableValue holds the key/value mapping of a Terraform variable.
 message VariableValue {
     string name = 1;
@@ -349,6 +354,7 @@ message PlanComplete {
     repeated Preset presets = 8;
     bytes plan = 9;
     bytes module_files = 10;
+    repeated ResourceReplacement resource_replacements = 11;
 }
 
 // ApplyRequest asks the provisioner to apply the changes.  Apply MUST be preceded by a successful plan request/response

site/e2e/helpers.ts

@@ -581,6 +581,7 @@ const createTemplateVersionTar = async (
 					externalAuthProviders: response.apply?.externalAuthProviders ?? [],
 					timings: response.apply?.timings ?? [],
 					presets: [],
+					resourceReplacements: [],
 					plan: emptyPlan,
 					moduleFiles: new Uint8Array(),
 				},
@@ -705,6 +706,7 @@ const createTemplateVersionTar = async (
 			timings: [],
 			modules: [],
 			presets: [],
+			resourceReplacements: [],
 			plan: emptyPlan,
 			moduleFiles: new Uint8Array(),
 			...response.plan,

site/e2e/provisionerGenerated.ts

@@ -120,6 +120,11 @@ export interface PresetParameter {
   value: string;
 }
 
+export interface ResourceReplacement {
+  resource: string;
+  paths: string[];
+}
+
 /** VariableValue holds the key/value mapping of a Terraform variable. */
 export interface VariableValue {
   name: string;
@@ -374,6 +379,7 @@ export interface PlanComplete {
   presets: Preset[];
   plan: Uint8Array;
   moduleFiles: Uint8Array;
+  resourceReplacements: ResourceReplacement[];
 }
 
 /**
@@ -573,6 +579,18 @@ export const PresetParameter = {
   },
 };
 
+export const ResourceReplacement = {
+  encode(message: ResourceReplacement, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.resource !== "") {
+      writer.uint32(10).string(message.resource);
+    }
+    for (const v of message.paths) {
+      writer.uint32(18).string(v!);
+    }
+    return writer;
+  },
+};
+
 export const VariableValue = {
   encode(message: VariableValue, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
     if (message.name !== "") {
@@ -1172,6 +1190,9 @@ export const PlanComplete = {
     if (message.moduleFiles.length !== 0) {
       writer.uint32(82).bytes(message.moduleFiles);
     }
+    for (const v of message.resourceReplacements) {
+      ResourceReplacement.encode(v!, writer.uint32(90).fork()).ldelim();
+    }
     return writer;
   },
 };