shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement expiration policy logic for prebuilds (#17996)

Browse Source 
## Summary 

This PR introduces support for expiration policies in prebuilds. The TTL
(time-to-live) is retrieved from the Terraform configuration
([terraform-provider-coder
PR](https://github.com/coder/terraform-provider-coder/pull/404)):
```
prebuilds = {
	  instances = 2
	  expiration_policy {
		  ttl = 86400
	  }
  }
```
**Note**: Since there is no need for precise TTL enforcement down to the
second, in this implementation expired prebuilds are handled in a single
reconciliation cycle: they are deleted, and new instances are created
only if needed to match the desired count.

## Changes

* The outcome of a reconciliation cycle is now expressed as a slice of
reconciliation actions, instead of a single aggregated action.
* Adjusted reconciliation logic to delete expired prebuilds and
guarantee that the number of desired instances is correct.
* Updated relevant data structures and methods to support expiration
policies parameters.
* Added documentation to `Prebuilt workspaces` page
* Update `terraform-provider-coder` to version 2.5.0:
https://github.com/coder/terraform-provider-coder/releases/tag/v2.5.0

Depends on: https://github.com/coder/terraform-provider-coder/pull/404
Fixes: https://github.com/coder/coder/issues/17916
This commit is contained in:
Susana Ferreira
2025-05-26 20:31:24 +01:00
committed by GitHub
parent 589f18627e
commit 6f6e73af03
18 changed files with 1503 additions and 994 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/queries.sql.go
+3
View File
@@ -6511,6 +6511,7 @@ SELECT
tvp.id,
tvp.name,
tvp.desired_instances AS desired_instances,
tvp.invalidate_after_secs AS ttl,
tvp.prebuild_status,
t.deleted,
t.deprecated != '' AS deprecated
@@ -6534,6 +6535,7 @@ type GetTemplatePresetsWithPrebuildsRow struct {
ID uuid.UUID `db:"id" json:"id"`
Name string `db:"name" json:"name"`
DesiredInstances sql.NullInt32 `db:"desired_instances" json:"desired_instances"`
Ttl sql.NullInt32 `db:"ttl" json:"ttl"`
PrebuildStatus PrebuildStatus `db:"prebuild_status" json:"prebuild_status"`
Deleted bool `db:"deleted" json:"deleted"`
Deprecated bool `db:"deprecated" json:"deprecated"`
@@ -6562,6 +6564,7 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa
&i.ID,
&i.Name,
&i.DesiredInstances,
&i.Ttl,
&i.PrebuildStatus,
&i.Deleted,
&i.Deprecated,
coderd/database/queries/prebuilds.sql
+1
View File
@@ -35,6 +35,7 @@ SELECT
tvp.id,
tvp.name,
tvp.desired_instances AS desired_instances,
tvp.invalidate_after_secs AS ttl,
tvp.prebuild_status,
t.deleted,
t.deprecated != '' AS deprecated
coderd/prebuilds/global_snapshot.go
+31 -1
View File
@@ -1,6 +1,8 @@
package prebuilds
import (
"time"
"github.com/google/uuid"
"golang.org/x/xerrors"
@@ -41,6 +43,7 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return nil, xerrors.Errorf("no preset found with ID %q", presetID)
}
// Only include workspaces that have successfully started
running := slice.Filter(s.RunningPrebuilds, func(prebuild database.GetRunningPrebuiltWorkspacesRow) bool {
if !prebuild.CurrentPresetID.Valid {
return false
@@ -48,6 +51,9 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return prebuild.CurrentPresetID.UUID == preset.ID
})
// Separate running workspaces into non-expired and expired based on the preset's TTL
nonExpired, expired := filterExpiredWorkspaces(preset, running)
inProgress := slice.Filter(s.PrebuildsInProgress, func(prebuild database.CountInProgressPrebuildsRow) bool {
return prebuild.PresetID.UUID == preset.ID
})
@@ -66,9 +72,33 @@ func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, err
return &PresetSnapshot{
Preset: preset,
Running: running,
Running: nonExpired,
Expired: expired,
InProgress: inProgress,
Backoff: backoffPtr,
IsHardLimited: isHardLimited,
}, nil
}
// filterExpiredWorkspaces splits running workspaces into expired and non-expired
// based on the preset's TTL.
// If TTL is missing or zero, all workspaces are considered non-expired.
func filterExpiredWorkspaces(preset database.GetTemplatePresetsWithPrebuildsRow, runningWorkspaces []database.GetRunningPrebuiltWorkspacesRow) (nonExpired []database.GetRunningPrebuiltWorkspacesRow, expired []database.GetRunningPrebuiltWorkspacesRow) {
if !preset.Ttl.Valid {
return runningWorkspaces, expired
}
ttl := time.Duration(preset.Ttl.Int32) * time.Second
if ttl <= 0 {
return runningWorkspaces, expired
}
for _, prebuild := range runningWorkspaces {
if time.Since(prebuild.CreatedAt) > ttl {
expired = append(expired, prebuild)
} else {
nonExpired = append(nonExpired, prebuild)
}
}
return nonExpired, expired
}
coderd/prebuilds/preset_snapshot.go
+77 -30
View File
@@ -31,9 +31,14 @@ const (
// PresetSnapshot is a filtered view of GlobalSnapshot focused on a single preset.
// It contains the raw data needed to calculate the current state of a preset's prebuilds,
// including running prebuilds, in-progress builds, and backoff information.
// - Running: prebuilds running and non-expired
// - Expired: prebuilds running and expired due to the preset's TTL
// - InProgress: prebuilds currently in progress
// - Backoff: holds failure info to decide if prebuild creation should be backed off
type PresetSnapshot struct {
Preset database.GetTemplatePresetsWithPrebuildsRow
Running []database.GetRunningPrebuiltWorkspacesRow
Expired []database.GetRunningPrebuiltWorkspacesRow
InProgress []database.CountInProgressPrebuildsRow
Backoff *database.GetPresetsBackoffRow
IsHardLimited bool
@@ -43,10 +48,11 @@ type PresetSnapshot struct {
// calculated from a PresetSnapshot. While PresetSnapshot contains raw data,
// ReconciliationState contains derived metrics that are directly used to
// determine what actions are needed (create, delete, or backoff).
// For example, it calculates how many prebuilds are eligible, how many are
// extraneous, and how many are in various transition states.
// For example, it calculates how many prebuilds are expired, eligible,
// how many are extraneous, and how many are in various transition states.
type ReconciliationState struct {
Actual int32 // Number of currently running prebuilds
Actual int32 // Number of currently running prebuilds, i.e., non-expired, expired and extraneous prebuilds
Expired int32 // Number of currently running prebuilds that exceeded their allowed time-to-live (TTL)
Desired int32 // Number of prebuilds desired as defined in the preset
Eligible int32 // Number of prebuilds that are ready to be claimed
Extraneous int32 // Number of extra running prebuilds beyond the desired count
@@ -78,7 +84,8 @@ func (ra *ReconciliationActions) IsNoop() bool {
}
// CalculateState computes the current state of prebuilds for a preset, including:
// - Actual: Number of currently running prebuilds
// - Actual: Number of currently running prebuilds, i.e., non-expired and expired prebuilds
// - Expired: Number of currently running expired prebuilds
// - Desired: Number of prebuilds desired as defined in the preset
// - Eligible: Number of prebuilds that are ready to be claimed
// - Extraneous: Number of extra running prebuilds beyond the desired count
@@ -92,23 +99,28 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
var (
actual int32
desired int32
expired int32
eligible int32
extraneous int32
)
// #nosec G115 - Safe conversion as p.Running slice length is expected to be within int32 range
actual = int32(len(p.Running))
// #nosec G115 - Safe conversion as p.Running and p.Expired slice length is expected to be within int32 range
actual = int32(len(p.Running) + len(p.Expired))
// #nosec G115 - Safe conversion as p.Expired slice length is expected to be within int32 range
expired = int32(len(p.Expired))
if p.isActive() {
desired = p.Preset.DesiredInstances.Int32
eligible = p.countEligible()
extraneous = max(actual-desired, 0)
extraneous = max(actual-expired-desired, 0)
}
starting, stopping, deleting := p.countInProgress()
return &ReconciliationState{
Actual: actual,
Expired: expired,
Desired: desired,
Eligible: eligible,
Extraneous: extraneous,
@@ -126,6 +138,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
// 3. For active presets, it calculates the number of prebuilds to create or delete based on:
// - The desired number of instances
// - Currently running prebuilds
// - Currently running expired prebuilds
// - Prebuilds in transition states (starting/stopping/deleting)
// - Any extraneous prebuilds that need to be removed
//
@@ -133,7 +146,7 @@ func (p PresetSnapshot) CalculateState() *ReconciliationState {
// - ActionTypeBackoff: Only BackoffUntil is set, indicating when to retry
// - ActionTypeCreate: Only Create is set, indicating how many prebuilds to create
// - ActionTypeDelete: Only DeleteIDs is set, containing IDs of prebuilds to delete
func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, error) {
func (p PresetSnapshot) CalculateActions(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, error) {
// TODO: align workspace states with how we represent them on the FE and the CLI
// right now there's some slight differences which can lead to additional prebuilds being created
@@ -158,45 +171,77 @@ func (p PresetSnapshot) isActive() bool {
return p.Preset.UsingActiveVersion && !p.Preset.Deleted && !p.Preset.Deprecated
}
// handleActiveTemplateVersion deletes excess prebuilds if there are too many,
// otherwise creates new ones to reach the desired count.
func (p PresetSnapshot) handleActiveTemplateVersion() (*ReconciliationActions, error) {
// handleActiveTemplateVersion determines the reconciliation actions for a preset with an active template version.
// It ensures the system moves towards the desired number of healthy prebuilds.
//
// The reconciliation follows this order:
// 1. Delete expired prebuilds: These are no longer valid and must be removed first.
// 2. Delete extraneous prebuilds: After expired ones are removed, if the number of running non-expired prebuilds
// still exceeds the desired count, the oldest prebuilds are deleted to reduce excess.
// 3. Create missing prebuilds: If the number of non-expired, non-starting prebuilds is still below the desired count,
// create the necessary number of prebuilds to reach the target.
//
// The function returns a list of actions to be executed to achieve the desired state.
func (p PresetSnapshot) handleActiveTemplateVersion() (actions []*ReconciliationActions, err error) {
state := p.CalculateState()
// If we have more prebuilds than desired, delete the oldest ones
if state.Extraneous > 0 {
return &ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: p.getOldestPrebuildIDs(int(state.Extraneous)),
}, nil
// If we have expired prebuilds, delete them
if state.Expired > 0 {
var deleteIDs []uuid.UUID
for _, expired := range p.Expired {
deleteIDs = append(deleteIDs, expired.ID)
}
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
})
}
// If we still have more prebuilds than desired, delete the oldest ones
if state.Extraneous > 0 {
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: p.getOldestPrebuildIDs(int(state.Extraneous)),
})
}
// Number of running prebuilds excluding the recently deleted Expired
runningValid := state.Actual - state.Expired
// Calculate how many new prebuilds we need to create
// We subtract starting prebuilds since they're already being created
prebuildsToCreate := max(state.Desired-state.Actual-state.Starting, 0)
prebuildsToCreate := max(state.Desired-runningValid-state.Starting, 0)
if prebuildsToCreate > 0 {
actions = append(actions,
&ReconciliationActions{
ActionType: ActionTypeCreate,
Create: prebuildsToCreate,
})
}
return &ReconciliationActions{
ActionType: ActionTypeCreate,
Create: prebuildsToCreate,
}, nil
return actions, nil
}
// handleInactiveTemplateVersion deletes all running prebuilds except those already being deleted
// to avoid duplicate deletion attempts.
func (p PresetSnapshot) handleInactiveTemplateVersion() (*ReconciliationActions, error) {
func (p PresetSnapshot) handleInactiveTemplateVersion() ([]*ReconciliationActions, error) {
prebuildsToDelete := len(p.Running)
deleteIDs := p.getOldestPrebuildIDs(prebuildsToDelete)
return &ReconciliationActions{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
return []*ReconciliationActions{
{
ActionType: ActionTypeDelete,
DeleteIDs: deleteIDs,
},
}, nil
}
// needsBackoffPeriod checks if we should delay prebuild creation due to recent failures.
// If there were failures, it calculates a backoff period based on the number of failures
// and returns true if we're still within that period.
func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) (*ReconciliationActions, bool) {
func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval time.Duration) ([]*ReconciliationActions, bool) {
if p.Backoff == nil || p.Backoff.NumFailed == 0 {
return nil, false
}
@@ -205,9 +250,11 @@ func (p PresetSnapshot) needsBackoffPeriod(clock quartz.Clock, backoffInterval t
return nil, false
}
return &ReconciliationActions{
ActionType: ActionTypeBackoff,
BackoffUntil: backoffUntil,
return []*ReconciliationActions{
{
ActionType: ActionTypeBackoff,
BackoffUntil: backoffUntil,
},
}, true
}
coderd/prebuilds/preset_snapshot_test.go
+300 -97
View File
@@ -23,6 +23,7 @@ type options struct {
presetName string
prebuiltWorkspaceID uuid.UUID
workspaceName string
ttl int32
}
// templateID is common across all option sets.
@@ -34,6 +35,7 @@ const (
optionSet0 = iota
optionSet1
optionSet2
optionSet3
)
var opts = map[uint]options{
@@ -61,6 +63,15 @@ var opts = map[uint]options{
prebuiltWorkspaceID: uuid.UUID{33},
workspaceName: "prebuilds2",
},
optionSet3: {
templateID: templateID,
templateVersionID: uuid.UUID{41},
presetID: uuid.UUID{42},
presetName: "my-preset",
prebuiltWorkspaceID: uuid.UUID{43},
workspaceName: "prebuilds3",
ttl: 5, // seconds
},
}
// A new template version with a preset without prebuilds configured should result in no prebuilds being created.
@@ -82,10 +93,7 @@ func TestNoPrebuilds(t *testing.T) {
require.NoError(t, err)
validateState(t, prebuilds.ReconciliationState{ /*all zero values*/ }, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 0,
}, *actions)
validateActions(t, nil, actions)
}
// A new template version with a preset with prebuilds configured should result in a new prebuild being created.
@@ -109,10 +117,12 @@ func TestNetNew(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
// A new template version is created with a preset with prebuilds configured; this outdates the older version and
@@ -149,10 +159,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
},
}, actions)
// WHEN: calculating the current preset's state.
ps, err = snapshot.FilterByPreset(current.presetID)
@@ -163,10 +175,12 @@ func TestOutdatedPrebuilds(t *testing.T) {
actions, err = ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
validateState(t, prebuilds.ReconciliationState{Desired: 1}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
// Make sure that outdated prebuild will be deleted, even if deletion of another outdated prebuild is already in progress.
@@ -214,10 +228,12 @@ func TestDeleteOutdatedPrebuilds(t *testing.T) {
Deleting: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{outdated.prebuiltWorkspaceID},
},
}, actions)
}
// A new template version is created with a preset with prebuilds configured; while a prebuild is provisioning up or down,
@@ -233,7 +249,7 @@ func TestInProgressActions(t *testing.T) {
desired int32
running int32
inProgress int32
checkFn func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions)
checkFn func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
}{
// With no running prebuilds and one starting, no creations/deletions should take place.
{
@@ -242,11 +258,9 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one running prebuild and one starting, no creations/deletions should occur since we're approaching the correct state.
@@ -256,11 +270,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 1,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one running prebuild and one starting, no creations/deletions should occur
@@ -271,11 +283,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Starting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one prebuild desired and one stopping, a new prebuild will be created.
@@ -285,11 +295,13 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@@ -300,11 +312,13 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 3, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@@ -315,11 +329,9 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 3,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 3, Desired: 3, Stopping: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With one prebuild desired and one deleting, a new prebuild will be created.
@@ -329,11 +341,13 @@ func TestInProgressActions(t *testing.T) {
desired: 1,
running: 0,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Desired: 1, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@@ -344,11 +358,13 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 1,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 2, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
},
},
@@ -359,11 +375,9 @@ func TestInProgressActions(t *testing.T) {
desired: 2,
running: 2,
inProgress: 1,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Deleting: 1}, state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
}, actions)
validateActions(t, nil, actions)
},
},
// With 3 prebuilds desired, 1 running, and 2 starting, no creations should occur since the builds are in progress.
@@ -373,9 +387,9 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 1,
inProgress: 2,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 1, Desired: 3, Starting: 2}, state)
validateActions(t, prebuilds.ReconciliationActions{ActionType: prebuilds.ActionTypeCreate, Create: 0}, actions)
validateActions(t, nil, actions)
},
},
// With 3 prebuilds desired, 5 running, and 2 deleting, no deletions should occur since the builds are in progress.
@@ -385,17 +399,20 @@ func TestInProgressActions(t *testing.T) {
desired: 3,
running: 5,
inProgress: 2,
checkFn: func(state prebuilds.ReconciliationState, actions prebuilds.ReconciliationActions) {
checkFn: func(state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 5, Desired: 3, Deleting: 2, Extraneous: 2}
expectedActions := prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
},
}
validateState(t, expectedState, state)
assert.EqualValuesf(t, expectedActions.ActionType, actions.ActionType, "'ActionType' did not match expectation")
assert.Len(t, actions.DeleteIDs, 2, "'deleteIDs' did not match expectation")
assert.EqualValuesf(t, expectedActions.Create, actions.Create, "'create' did not match expectation")
assert.EqualValuesf(t, expectedActions.BackoffUntil, actions.BackoffUntil, "'BackoffUntil' did not match expectation")
require.Equal(t, len(expectedActions), len(actions))
assert.EqualValuesf(t, expectedActions[0].ActionType, actions[0].ActionType, "'ActionType' did not match expectation")
assert.Len(t, actions[0].DeleteIDs, 2, "'deleteIDs' did not match expectation")
assert.EqualValuesf(t, expectedActions[0].Create, actions[0].Create, "'create' did not match expectation")
assert.EqualValuesf(t, expectedActions[0].BackoffUntil, actions[0].BackoffUntil, "'BackoffUntil' did not match expectation")
},
},
}
@@ -450,7 +467,7 @@ func TestInProgressActions(t *testing.T) {
state := ps.CalculateState()
actions, err := ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
tc.checkFn(*state, *actions)
tc.checkFn(*state, actions)
})
}
}
@@ -496,10 +513,187 @@ func TestExtraneous(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 2, Desired: 1, Extraneous: 1, Eligible: 2,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{older},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{older},
},
}, actions)
}
// A prebuild is considered Expired when it has exceeded their time-to-live (TTL)
// specified in the preset's cache invalidation invalidate_after_secs parameter.
func TestExpiredPrebuilds(t *testing.T) {
t.Parallel()
current := opts[optionSet3]
clock := quartz.NewMock(t)
cases := []struct {
name string
running int32
desired int32
expired int32
checkFn func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions)
}{
// With 2 running prebuilds, none of which are expired, and the desired count is met,
// no deletions or creations should occur.
{
name: "no expired prebuilds - no actions taken",
running: 2,
desired: 2,
expired: 0,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
validateState(t, prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 0}, state)
validateActions(t, nil, actions)
},
},
// With 2 running prebuilds, 1 of which is expired, the expired prebuild should be deleted,
// and one new prebuild should be created to maintain the desired count.
{
name: "one expired prebuild – deleted and replaced",
running: 2,
desired: 2,
expired: 1,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 1}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID},
},
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 2 running prebuilds, both expired, both should be deleted,
// and 2 new prebuilds created to match the desired count.
{
name: "all prebuilds expired – all deleted and recreated",
running: 2,
desired: 2,
expired: 2,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 2, Desired: 2, Expired: 2}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
},
{
ActionType: prebuilds.ActionTypeCreate,
Create: 2,
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 4 running prebuilds, 2 of which are expired, and the desired count is 2,
// the expired prebuilds should be deleted. No new creations are needed
// since removing the expired ones brings actual = desired.
{
name: "expired prebuilds deleted to reach desired count",
running: 4,
desired: 2,
expired: 2,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 2, Extraneous: 0}
expectedActions := []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID, runningPrebuilds[1].ID},
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
// With 4 running prebuilds (1 expired), and the desired count is 2,
// the first action should delete the expired one,
// and the second action should delete one additional (non-expired) prebuild
// to eliminate the remaining excess.
{
name: "expired prebuild deleted first, then extraneous",
running: 4,
desired: 2,
expired: 1,
checkFn: func(runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow, state prebuilds.ReconciliationState, actions []*prebuilds.ReconciliationActions) {
expectedState := prebuilds.ReconciliationState{Actual: 4, Desired: 2, Expired: 1, Extraneous: 1}
expectedActions := []*prebuilds.ReconciliationActions{
// First action correspond to deleting the expired prebuild,
// and the second action corresponds to deleting the extraneous prebuild
// corresponding to the oldest one after the expired prebuild
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[0].ID},
},
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{runningPrebuilds[1].ID},
},
}
validateState(t, expectedState, state)
validateActions(t, expectedActions, actions)
},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
// GIVEN: a preset.
defaultPreset := preset(true, tc.desired, current)
presets := []database.GetTemplatePresetsWithPrebuildsRow{
defaultPreset,
}
// GIVEN: running prebuilt workspaces for the preset.
running := make([]database.GetRunningPrebuiltWorkspacesRow, 0, tc.running)
expiredCount := 0
ttlDuration := time.Duration(defaultPreset.Ttl.Int32)
for range tc.running {
name, err := prebuilds.GenerateName()
require.NoError(t, err)
prebuildCreateAt := time.Now()
if int(tc.expired) > expiredCount {
// Update the prebuild workspace createdAt to exceed its TTL (5 seconds)
prebuildCreateAt = prebuildCreateAt.Add(-ttlDuration - 10*time.Second)
expiredCount++
}
running = append(running, database.GetRunningPrebuiltWorkspacesRow{
ID: uuid.New(),
Name: name,
TemplateID: current.templateID,
TemplateVersionID: current.templateVersionID,
CurrentPresetID: uuid.NullUUID{UUID: current.presetID, Valid: true},
Ready: false,
CreatedAt: prebuildCreateAt,
})
}
// WHEN: calculating the current preset's state.
snapshot := prebuilds.NewGlobalSnapshot(presets, running, nil, nil, nil)
ps, err := snapshot.FilterByPreset(current.presetID)
require.NoError(t, err)
// THEN: we should identify that this prebuild is expired.
state := ps.CalculateState()
actions, err := ps.CalculateActions(clock, backoffInterval)
require.NoError(t, err)
tc.checkFn(running, *state, actions)
})
}
}
// A template marked as deprecated will not have prebuilds running.
@@ -536,10 +730,12 @@ func TestDeprecated(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{current.prebuiltWorkspaceID},
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeDelete,
DeleteIDs: []uuid.UUID{current.prebuiltWorkspaceID},
},
}, actions)
}
// If the latest build failed, backoff exponentially with the given interval.
@@ -587,10 +783,12 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 0, Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeBackoff,
BackoffUntil: lastBuildTime.Add(time.Duration(numFailed) * backoffInterval),
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeBackoff,
BackoffUntil: lastBuildTime.Add(time.Duration(numFailed) * backoffInterval),
},
}, actions)
// WHEN: calculating the other preset's state.
psOther, err := snapshot.FilterByPreset(other.presetID)
@@ -603,10 +801,7 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 1, Desired: 1, Eligible: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
BackoffUntil: time.Time{},
}, *actions)
validateActions(t, nil, actions)
// WHEN: the clock is advanced a backoff interval.
clock.Advance(backoffInterval + time.Microsecond)
@@ -620,11 +815,12 @@ func TestLatestBuildFailed(t *testing.T) {
validateState(t, prebuilds.ReconciliationState{
Actual: 0, Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1, // <--- NOTE: we're now able to create a new prebuild because the interval has elapsed.
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1, // <--- NOTE: we're now able to create a new prebuild because the interval has elapsed.
},
}, actions)
}
func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
@@ -684,10 +880,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
Starting: 1,
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 0,
}, *actions)
validateActions(t, nil, actions)
}
// One prebuild has to be created for preset 2. Make sure preset 1 doesn't block preset 2.
@@ -703,14 +896,23 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
Starting: 0,
Desired: 1,
}, *state)
validateActions(t, prebuilds.ReconciliationActions{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
}, *actions)
validateActions(t, []*prebuilds.ReconciliationActions{
{
ActionType: prebuilds.ActionTypeCreate,
Create: 1,
},
}, actions)
}
}
func preset(active bool, instances int32, opts options, muts ...func(row database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow) database.GetTemplatePresetsWithPrebuildsRow {
ttl := sql.NullInt32{}
if opts.ttl > 0 {
ttl = sql.NullInt32{
Valid: true,
Int32: opts.ttl,
}
}
entry := database.GetTemplatePresetsWithPrebuildsRow{
TemplateID: opts.templateID,
TemplateVersionID: opts.templateVersionID,
@@ -723,6 +925,7 @@ func preset(active bool, instances int32, opts options, muts ...func(row databas
},
Deleted: false,
Deprecated: false,
Ttl: ttl,
}
for _, mut := range muts {
@@ -758,6 +961,6 @@ func validateState(t *testing.T, expected, actual prebuilds.ReconciliationState)
// validateActions is a convenience func to make tests more readable; it exploits the fact that the default states for
// prebuilds align with zero values.
func validateActions(t *testing.T, expected, actual prebuilds.ReconciliationActions) {
func validateActions(t *testing.T, expected, actual []*prebuilds.ReconciliationActions) {
require.Equal(t, expected, actual)
}
coderd/provisionerdserver/provisionerdserver.go
+13 -10
View File
@@ -2059,23 +2059,26 @@ func InsertWorkspacePresetsAndParameters(ctx context.Context, logger slog.Logger
func InsertWorkspacePresetAndParameters(ctx context.Context, db database.Store, templateVersionID uuid.UUID, protoPreset *sdkproto.Preset, t time.Time) error {
err := db.InTx(func(tx database.Store) error {
var desiredInstances sql.NullInt32
var desiredInstances, ttl sql.NullInt32
if protoPreset != nil && protoPreset.Prebuild != nil {
desiredInstances = sql.NullInt32{
Int32: protoPreset.Prebuild.Instances,
Valid: true,
}
if protoPreset.Prebuild.ExpirationPolicy != nil {
ttl = sql.NullInt32{
Int32: protoPreset.Prebuild.ExpirationPolicy.Ttl,
Valid: true,
}
}
}
dbPreset, err := tx.InsertPreset(ctx, database.InsertPresetParams{
ID: uuid.New(),
TemplateVersionID: templateVersionID,
Name: protoPreset.Name,
CreatedAt: t,
DesiredInstances: desiredInstances,
InvalidateAfterSecs: sql.NullInt32{
Int32: 0,
Valid: false,
}, // TODO: implement cache invalidation
ID: uuid.New(),
TemplateVersionID: templateVersionID,
Name: protoPreset.Name,
CreatedAt: t,
DesiredInstances: desiredInstances,
InvalidateAfterSecs: ttl,
})
if err != nil {
return xerrors.Errorf("insert preset: %w", err)
docs/admin/templates/extending-templates/prebuilt-workspaces.md
+18 -2
View File
@@ -31,7 +31,8 @@ Prebuilt workspaces are tightly integrated with [workspace presets](./parameters
## Enable prebuilt workspaces for template presets
In your template, add a `prebuilds` block within a `coder_workspace_preset` definition to identify the number of prebuilt
instances your Coder deployment should maintain:
instances your Coder deployment should maintain, and optionally configure a `expiration_policy` block to set a TTL
(Time To Live) for unclaimed prebuilt workspaces to ensure stale resources are automatically cleaned up.
```hcl
data "coder_workspace_preset" "goland" {
@@ -42,7 +43,10 @@ instances your Coder deployment should maintain:
memory = 16
}
prebuilds {
instances = 3 # Number of prebuilt workspaces to maintain
instances = 3 # Number of prebuilt workspaces to maintain
expiration_policy {
ttl = 86400 # Time (in seconds) after which unclaimed prebuilds are expired (1 day)
}
}
}
```
@@ -50,6 +54,9 @@ instances your Coder deployment should maintain:
After you publish a new template version, Coder will automatically provision and maintain prebuilt workspaces through an
internal reconciliation loop (similar to Kubernetes) to ensure the defined `instances` count are running.
The `expiration_policy` block ensures that any prebuilt workspaces left unclaimed for more than `ttl` seconds is considered
expired and automatically cleaned up.
## Prebuilt workspace lifecycle
Prebuilt workspaces follow a specific lifecycle from creation through eligibility to claiming.
@@ -95,6 +102,15 @@ Unclaimed prebuilt workspaces can be interacted with in the same way as any othe
However, if a Prebuilt workspace is stopped, the reconciliation loop will not destroy it.
This gives template admins the ability to park problematic prebuilt workspaces in a stopped state for further investigation.
### Expiration Policy
Prebuilt workspaces support expiration policies through the `ttl` setting inside the `expiration_policy` block.
This value defines the Time To Live (TTL) of a prebuilt workspace, i.e., the duration in seconds that an unclaimed
prebuilt workspace can remain before it is considered expired and eligible for cleanup.
Expired prebuilt workspaces are removed during the reconciliation loop to avoid stale environments and resource waste.
New prebuilt workspaces are only created to maintain the desired count if needed.
### Template updates and the prebuilt workspace lifecycle
Prebuilt workspaces are not updated after they are provisioned.
enterprise/coderd/prebuilds/reconcile.go
+106 -80
View File
@@ -396,97 +396,28 @@ func (c *StoreReconciler) ReconcilePreset(ctx context.Context, ps prebuilds.Pres
actions, err := c.CalculateActions(ctx, ps)
if err != nil {
logger.Error(ctx, "failed to calculate actions for preset", slog.Error(err))
return nil
}
// Nothing has to be done.
if !ps.Preset.UsingActiveVersion && actions.IsNoop() {
logger.Debug(ctx, "skipping reconciliation for preset - nothing has to be done")
return nil
}
// nolint:gocritic // ReconcilePreset needs Prebuilds Orchestrator permissions.
prebuildsCtx := dbauthz.AsPrebuildsOrchestrator(ctx)
levelFn := logger.Debug
switch {
case actions.ActionType == prebuilds.ActionTypeBackoff:
levelFn = logger.Warn
// Log at info level when there's a change to be effected.
case actions.ActionType == prebuilds.ActionTypeCreate && actions.Create > 0:
levelFn = logger.Info
case actions.ActionType == prebuilds.ActionTypeDelete && len(actions.DeleteIDs) > 0:
levelFn = logger.Info
return err
}
fields := []any{
slog.F("action_type", actions.ActionType),
slog.F("create_count", actions.Create), slog.F("delete_count", len(actions.DeleteIDs)),
slog.F("to_delete", actions.DeleteIDs),
slog.F("desired", state.Desired), slog.F("actual", state.Actual),
slog.F("extraneous", state.Extraneous), slog.F("starting", state.Starting),
slog.F("stopping", state.Stopping), slog.F("deleting", state.Deleting),
slog.F("eligible", state.Eligible),
}
levelFn(ctx, "calculated reconciliation actions for preset", fields...)
levelFn := logger.Debug
levelFn(ctx, "calculated reconciliation state for preset", fields...)
switch actions.ActionType {
case prebuilds.ActionTypeBackoff:
// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
levelFn(ctx, "template prebuild state retrieved, backing off",
append(fields,
slog.F("backoff_until", actions.BackoffUntil.Format(time.RFC3339)),
slog.F("backoff_secs", math.Round(actions.BackoffUntil.Sub(c.clock.Now()).Seconds())),
)...)
return nil
case prebuilds.ActionTypeCreate:
// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
// This is obviously not comprehensive protection against this sort of problem, but this is one essential check.
desired := ps.Preset.DesiredInstances.Int32
if actions.Create > desired {
logger.Critical(ctx, "determined excessive count of prebuilds to create; clamping to desired count",
slog.F("create_count", actions.Create), slog.F("desired_count", desired))
actions.Create = desired
var multiErr multierror.Error
for _, action := range actions {
err = c.executeReconciliationAction(ctx, logger, ps, action)
if err != nil {
logger.Error(ctx, "failed to execute action", "type", action.ActionType, slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
// If preset is hard-limited, and it's a create operation, log it and exit early.
// Creation operation is disallowed for hard-limited preset.
if ps.IsHardLimited && actions.Create > 0 {
logger.Warn(ctx, "skipping hard limited preset for create operation")
return nil
}
var multiErr multierror.Error
for range actions.Create {
if err := c.createPrebuiltWorkspace(prebuildsCtx, uuid.New(), ps.Preset.TemplateID, ps.Preset.ID); err != nil {
logger.Error(ctx, "failed to create prebuild", slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
}
return multiErr.ErrorOrNil()
case prebuilds.ActionTypeDelete:
var multiErr multierror.Error
for _, id := range actions.DeleteIDs {
if err := c.deletePrebuiltWorkspace(prebuildsCtx, id, ps.Preset.TemplateID, ps.Preset.ID); err != nil {
logger.Error(ctx, "failed to delete prebuild", slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
}
return multiErr.ErrorOrNil()
default:
return xerrors.Errorf("unknown action type: %v", actions.ActionType)
}
return multiErr.ErrorOrNil()
}
func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context, ps prebuilds.PresetSnapshot) error {
@@ -532,7 +463,7 @@ func (c *StoreReconciler) notifyPrebuildFailureLimitReached(ctx context.Context,
return nil
}
func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) (*prebuilds.ReconciliationActions, error) {
func (c *StoreReconciler) CalculateActions(ctx context.Context, snapshot prebuilds.PresetSnapshot) ([]*prebuilds.ReconciliationActions, error) {
if ctx.Err() != nil {
return nil, ctx.Err()
}
@@ -581,6 +512,101 @@ func (c *StoreReconciler) WithReconciliationLock(
})
}
// executeReconciliationAction executes a reconciliation action on the given preset snapshot.
//
// The action can be of different types (create, delete, backoff), and may internally include
// multiple items to process, for example, a delete action can contain multiple prebuild IDs to delete,
// and a create action includes a count of prebuilds to create.
//
// This method handles logging at appropriate levels and performs the necessary operations
// according to the action type. It returns an error if any part of the action fails.
func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logger slog.Logger, ps prebuilds.PresetSnapshot, action *prebuilds.ReconciliationActions) error {
levelFn := logger.Debug
// Nothing has to be done.
if !ps.Preset.UsingActiveVersion && action.IsNoop() {
logger.Debug(ctx, "skipping reconciliation for preset - nothing has to be done",
slog.F("template_id", ps.Preset.TemplateID.String()), slog.F("template_name", ps.Preset.TemplateName),
slog.F("template_version_id", ps.Preset.TemplateVersionID.String()), slog.F("template_version_name", ps.Preset.TemplateVersionName),
slog.F("preset_id", ps.Preset.ID.String()), slog.F("preset_name", ps.Preset.Name))
return nil
}
// nolint:gocritic // ReconcilePreset needs Prebuilds Orchestrator permissions.
prebuildsCtx := dbauthz.AsPrebuildsOrchestrator(ctx)
fields := []any{
slog.F("action_type", action.ActionType), slog.F("create_count", action.Create),
slog.F("delete_count", len(action.DeleteIDs)), slog.F("to_delete", action.DeleteIDs),
}
levelFn(ctx, "calculated reconciliation action for preset", fields...)
switch {
case action.ActionType == prebuilds.ActionTypeBackoff:
levelFn = logger.Warn
// Log at info level when there's a change to be effected.
case action.ActionType == prebuilds.ActionTypeCreate && action.Create > 0:
levelFn = logger.Info
case action.ActionType == prebuilds.ActionTypeDelete && len(action.DeleteIDs) > 0:
levelFn = logger.Info
}
switch action.ActionType {
case prebuilds.ActionTypeBackoff:
// If there is anything to backoff for (usually a cycle of failed prebuilds), then log and bail out.
levelFn(ctx, "template prebuild state retrieved, backing off",
append(fields,
slog.F("backoff_until", action.BackoffUntil.Format(time.RFC3339)),
slog.F("backoff_secs", math.Round(action.BackoffUntil.Sub(c.clock.Now()).Seconds())),
)...)
return nil
case prebuilds.ActionTypeCreate:
// Unexpected things happen (i.e. bugs or bitflips); let's defend against disastrous outcomes.
// See https://blog.robertelder.org/causes-of-bit-flips-in-computer-memory/.
// This is obviously not comprehensive protection against this sort of problem, but this is one essential check.
desired := ps.Preset.DesiredInstances.Int32
if action.Create > desired {
logger.Critical(ctx, "determined excessive count of prebuilds to create; clamping to desired count",
slog.F("create_count", action.Create), slog.F("desired_count", desired))
action.Create = desired
}
// If preset is hard-limited, and it's a create operation, log it and exit early.
// Creation operation is disallowed for hard-limited preset.
if ps.IsHardLimited && action.Create > 0 {
logger.Warn(ctx, "skipping hard limited preset for create operation")
return nil
}
var multiErr multierror.Error
for range action.Create {
if err := c.createPrebuiltWorkspace(prebuildsCtx, uuid.New(), ps.Preset.TemplateID, ps.Preset.ID); err != nil {
logger.Error(ctx, "failed to create prebuild", slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
}
return multiErr.ErrorOrNil()
case prebuilds.ActionTypeDelete:
var multiErr multierror.Error
for _, id := range action.DeleteIDs {
if err := c.deletePrebuiltWorkspace(prebuildsCtx, id, ps.Preset.TemplateID, ps.Preset.ID); err != nil {
logger.Error(ctx, "failed to delete prebuild", slog.Error(err))
multiErr.Errors = append(multiErr.Errors, err)
}
}
return multiErr.ErrorOrNil()
default:
return xerrors.Errorf("unknown action type: %v", action.ActionType)
}
}
func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error {
name, err := prebuilds.GenerateName()
if err != nil {
enterprise/coderd/prebuilds/reconcile_test.go
+16 -9
View File
@@ -1226,17 +1226,18 @@ func TestFailedBuildBackoff(t *testing.T) {
state := presetState.CalculateState()
actions, err := reconciler.CalculateActions(ctx, *presetState)
require.NoError(t, err)
require.Equal(t, 1, len(actions))
// Then: the backoff time is in the future, no prebuilds are running, and we won't create any new prebuilds.
require.EqualValues(t, 0, state.Actual)
require.EqualValues(t, 0, actions.Create)
require.EqualValues(t, 0, actions[0].Create)
require.EqualValues(t, desiredInstances, state.Desired)
require.True(t, clock.Now().Before(actions.BackoffUntil))
require.True(t, clock.Now().Before(actions[0].BackoffUntil))
// Then: the backoff time is as expected based on the number of failed builds.
require.NotNil(t, presetState.Backoff)
require.EqualValues(t, desiredInstances, presetState.Backoff.NumFailed)
require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions.BackoffUntil).Truncate(backoffInterval))
require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions[0].BackoffUntil).Truncate(backoffInterval))
// When: advancing to the next tick which is still within the backoff time.
clock.Advance(cfg.ReconciliationInterval.Value())
@@ -1249,13 +1250,15 @@ func TestFailedBuildBackoff(t *testing.T) {
newState := presetState.CalculateState()
newActions, err := reconciler.CalculateActions(ctx, *presetState)
require.NoError(t, err)
require.Equal(t, 1, len(newActions))
require.EqualValues(t, 0, newState.Actual)
require.EqualValues(t, 0, newActions.Create)
require.EqualValues(t, 0, newActions[0].Create)
require.EqualValues(t, desiredInstances, newState.Desired)
require.EqualValues(t, actions.BackoffUntil, newActions.BackoffUntil)
require.EqualValues(t, actions[0].BackoffUntil, newActions[0].BackoffUntil)
// When: advancing beyond the backoff time.
clock.Advance(clock.Until(actions.BackoffUntil.Add(time.Second)))
clock.Advance(clock.Until(actions[0].BackoffUntil.Add(time.Second)))
// Then: we will attempt to create a new prebuild.
snapshot, err = reconciler.SnapshotState(ctx, db)
@@ -1265,9 +1268,11 @@ func TestFailedBuildBackoff(t *testing.T) {
state = presetState.CalculateState()
actions, err = reconciler.CalculateActions(ctx, *presetState)
require.NoError(t, err)
require.Equal(t, 1, len(actions))
require.EqualValues(t, 0, state.Actual)
require.EqualValues(t, desiredInstances, state.Desired)
require.EqualValues(t, desiredInstances, actions.Create)
require.EqualValues(t, desiredInstances, actions[0].Create)
// When: the desired number of new prebuild are provisioned, but one fails again.
for i := 0; i < desiredInstances; i++ {
@@ -1286,11 +1291,13 @@ func TestFailedBuildBackoff(t *testing.T) {
state = presetState.CalculateState()
actions, err = reconciler.CalculateActions(ctx, *presetState)
require.NoError(t, err)
require.Equal(t, 1, len(actions))
require.EqualValues(t, 1, state.Actual)
require.EqualValues(t, desiredInstances, state.Desired)
require.EqualValues(t, 0, actions.Create)
require.EqualValues(t, 0, actions[0].Create)
require.EqualValues(t, 3, presetState.Backoff.NumFailed)
require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions.BackoffUntil).Truncate(backoffInterval))
require.EqualValues(t, backoffInterval*time.Duration(presetState.Backoff.NumFailed), clock.Until(actions[0].BackoffUntil).Truncate(backoffInterval))
}
func TestReconciliationLock(t *testing.T) {
provisioner/terraform/resources.go
+8 -1
View File
@@ -897,14 +897,21 @@ func ConvertState(ctx context.Context, modules []*tfjson.StateModule, rawGraph s
)
}
var prebuildInstances int32
var expirationPolicy *proto.ExpirationPolicy
if len(preset.Prebuilds) > 0 {
prebuildInstances = int32(math.Min(math.MaxInt32, float64(preset.Prebuilds[0].Instances)))
if len(preset.Prebuilds[0].ExpirationPolicy) > 0 {
expirationPolicy = &proto.ExpirationPolicy{
Ttl: int32(math.Min(math.MaxInt32, float64(preset.Prebuilds[0].ExpirationPolicy[0].TTL))),
}
}
}
protoPreset := &proto.Preset{
Name: preset.Name,
Parameters: presetParameters,
Prebuild: &proto.Prebuild{
Instances: prebuildInstances,
Instances: prebuildInstances,
ExpirationPolicy: expirationPolicy,
},
}
provisioner/terraform/resources_test.go
+4
View File
@@ -786,6 +786,7 @@ func TestConvertResources(t *testing.T) {
Name: "dev",
OperatingSystem: "windows",
Architecture: "arm64",
ApiKeyScope: "all",
Auth: &proto.Agent_Token{},
ConnectionTimeoutSeconds: 120,
DisplayApps: &displayApps,
@@ -830,6 +831,9 @@ func TestConvertResources(t *testing.T) {
}},
Prebuild: &proto.Prebuild{
Instances: 4,
ExpirationPolicy: &proto.ExpirationPolicy{
Ttl: 86400,
},
},
}},
},
provisioner/terraform/testdata/resources/presets/presets.tf
Vendored
+3
View File
@@ -25,6 +25,9 @@ data "coder_workspace_preset" "MyFirstProject" {
}
prebuilds {
instances = 4
expiration_policy {
ttl = 86400
}
}
}
provisioner/terraform/testdata/resources/presets/presets.tfplan.json
Generated Vendored
+29 -1
View File
@@ -12,6 +12,7 @@
"provider_name": "registry.terraform.io/coder/coder",
"schema_version": 1,
"values": {
"api_key_scope": "all",
"arch": "arm64",
"auth": "token",
"connection_timeout": 120,
@@ -62,6 +63,7 @@
],
"before": null,
"after": {
"api_key_scope": "all",
"arch": "arm64",
"auth": "token",
"connection_timeout": 120,
@@ -134,6 +136,7 @@
"description": "blah blah",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "57ccea62-8edf-41d1-a2c1-33f365e27567",
"mutable": false,
@@ -141,6 +144,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ok"
@@ -164,6 +168,11 @@
},
"prebuilds": [
{
"expiration_policy": [
{
"ttl": 86400
}
],
"instances": 4
}
]
@@ -171,7 +180,11 @@
"sensitive_values": {
"parameters": {},
"prebuilds": [
{}
{
"expiration_policy": [
{}
]
}
]
}
}
@@ -191,6 +204,7 @@
"description": "First parameter from module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "1774175f-0efd-4a79-8d40-dbbc559bf7c1",
"mutable": true,
@@ -198,6 +212,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "abcdef"
@@ -218,6 +233,7 @@
"description": "Second parameter from module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "23d6841f-bb95-42bb-b7ea-5b254ce6c37d",
"mutable": true,
@@ -225,6 +241,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ghijkl"
@@ -250,6 +267,7 @@
"description": "First parameter from child module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "9d629df2-9846-47b2-ab1f-e7c882f35117",
"mutable": true,
@@ -257,6 +275,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "abcdef"
@@ -277,6 +296,7 @@
"description": "Second parameter from child module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "52ca7b77-42a1-4887-a2f5-7a728feebdd5",
"mutable": true,
@@ -284,6 +304,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ghijkl"
@@ -388,6 +409,13 @@
},
"prebuilds": [
{
"expiration_policy": [
{
"ttl": {
"constant_value": 86400
}
}
],
"instances": {
"constant_value": 4
}
provisioner/terraform/testdata/resources/presets/presets.tfstate.json
Generated Vendored
+21 -1
View File
@@ -16,6 +16,7 @@
"description": "blah blah",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "491d202d-5658-40d9-9adc-fd3a67f6042b",
"mutable": false,
@@ -23,6 +24,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ok"
@@ -46,6 +48,11 @@
},
"prebuilds": [
{
"expiration_policy": [
{
"ttl": 86400
}
],
"instances": 4
}
]
@@ -53,7 +60,11 @@
"sensitive_values": {
"parameters": {},
"prebuilds": [
{}
{
"expiration_policy": [
{}
]
}
]
}
},
@@ -65,6 +76,7 @@
"provider_name": "registry.terraform.io/coder/coder",
"schema_version": 1,
"values": {
"api_key_scope": "all",
"arch": "arm64",
"auth": "token",
"connection_timeout": 120,
@@ -133,6 +145,7 @@
"description": "First parameter from module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "0a4d1299-b174-43b0-91ad-50c1ca9a4c25",
"mutable": true,
@@ -140,6 +153,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "abcdef"
@@ -160,6 +174,7 @@
"description": "Second parameter from module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "f0812474-29fd-4c3c-ab40-9e66e36d4017",
"mutable": true,
@@ -167,6 +182,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ghijkl"
@@ -192,6 +208,7 @@
"description": "First parameter from child module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "27b5fae3-7671-4e61-bdfe-c940627a21b8",
"mutable": true,
@@ -199,6 +216,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "abcdef"
@@ -219,6 +237,7 @@
"description": "Second parameter from child module",
"display_name": null,
"ephemeral": false,
"form_type": "input",
"icon": null,
"id": "d285bb17-27ff-4a49-a12b-28582264b4d9",
"mutable": true,
@@ -226,6 +245,7 @@
"option": null,
"optional": true,
"order": null,
"styling": "{}",
"type": "string",
"validation": [],
"value": "ghijkl"
provisionerd/proto/version.go
+2
View File
@@ -25,6 +25,8 @@ import "github.com/coder/coder/v2/apiversion"
// - Add previous parameter values to 'WorkspaceBuild' jobs. Provisioner passes
// the previous values for the `terraform apply` to enforce monotonicity
// in the terraform provider.
// - Add new field named `expiration_policy` to `Prebuild`, with a field named
// `ttl` to define TTL-based expiration for unclaimed prebuilds.
const (
CurrentMajor = 1
CurrentMinor = 6
provisionersdk/proto/provisioner.pb.go
Generated
+841 -762
View File
File diff suppressed because it is too large Load Diff
provisionersdk/proto/provisioner.proto
+8
View File
@@ -57,8 +57,16 @@ message RichParameterValue {
string value = 2;
}
// ExpirationPolicy defines the policy for expiring unclaimed prebuilds.
// If a prebuild remains unclaimed for longer than ttl seconds, it is deleted and
// recreated to prevent staleness.
message ExpirationPolicy {
int32 ttl = 1;
}
message Prebuild {
int32 instances = 1;
ExpirationPolicy expiration_policy = 2;
}
// Preset represents a set of preset parameters for a template version.
site/e2e/provisionerGenerated.ts
Generated
+22
View File
@@ -104,8 +104,18 @@ export interface RichParameterValue {
value: string;
}
/**
* ExpirationPolicy defines the policy for expiring unclaimed prebuilds.
* If a prebuild remains unclaimed for longer than ttl seconds, it is deleted and
* recreated to prevent staleness.
*/
export interface ExpirationPolicy {
ttl: number;
}
export interface Prebuild {
instances: number;
expirationPolicy: ExpirationPolicy | undefined;
}
/** Preset represents a set of preset parameters for a template version. */
@@ -544,11 +554,23 @@ export const RichParameterValue = {
},
};
export const ExpirationPolicy = {
encode(message: ExpirationPolicy, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
if (message.ttl !== 0) {
writer.uint32(8).int32(message.ttl);
}
return writer;
},
};
export const Prebuild = {
encode(message: Prebuild, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
if (message.instances !== 0) {
writer.uint32(8).int32(message.instances);
}
if (message.expirationPolicy !== undefined) {
ExpirationPolicy.encode(message.expirationPolicy, writer.uint32(18).fork()).ldelim();
}
return writer;
},
};