feat: notifications: report failed workspace builds (#14571)

2026-06-02 20:48:20 +00:00 · 2024-09-18 09:11:44 +02:00
parent 1e5438eadb
commit 6de59371ea
29 changed files with 1545 additions and 55 deletions
@@ -56,6 +56,7 @@ import (
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
 	"github.com/coder/coder/v2/coderd/entitlements"
+	"github.com/coder/coder/v2/coderd/notifications/reports"
 	"github.com/coder/coder/v2/coderd/runtimeconfig"
 	"github.com/coder/pretty"
 	"github.com/coder/quartz"
@@ -1018,6 +1019,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.

 				// nolint:gocritic // TODO: create own role.
 				notificationsManager.Run(dbauthz.AsSystemRestricted(ctx))
+
+				// Run report generator to distribute periodic reports.
+				notificationReportGenerator := reports.NewReportGenerator(ctx, logger, options.Database, options.NotificationsEnqueuer, quartz.NewReal())
+				defer notificationReportGenerator.Close()
 			}

 			// Wrap the server in middleware that redirects to the access URL if
@@ -1459,6 +1459,13 @@ func (q *querier) GetExternalAuthLinksByUserID(ctx context.Context, userID uuid.
 	return fetchWithPostFilter(q.auth, policy.ActionReadPersonal, q.db.GetExternalAuthLinksByUserID)(ctx, userID)
 }

+func (q *querier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
+		return nil, err
+	}
+	return q.db.GetFailedWorkspaceBuildsByTemplateID(ctx, arg)
+}
+
 func (q *querier) GetFileByHashAndCreator(ctx context.Context, arg database.GetFileByHashAndCreatorParams) (database.File, error) {
 	file, err := q.db.GetFileByHashAndCreator(ctx, arg)
 	if err != nil {
@@ -1628,6 +1635,13 @@ func (q *querier) GetNotificationMessagesByStatus(ctx context.Context, arg datab
 	return q.db.GetNotificationMessagesByStatus(ctx, arg)
 }

+func (q *querier) GetNotificationReportGeneratorLogByTemplate(ctx context.Context, arg uuid.UUID) (database.NotificationReportGeneratorLog, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
+		return database.NotificationReportGeneratorLog{}, err
+	}
+	return q.db.GetNotificationReportGeneratorLogByTemplate(ctx, arg)
+}
+
 func (q *querier) GetNotificationTemplateByID(ctx context.Context, id uuid.UUID) (database.NotificationTemplate, error) {
 	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceNotificationTemplate); err != nil {
 		return database.NotificationTemplate{}, err
@@ -2510,6 +2524,13 @@ func (q *querier) GetWorkspaceBuildParameters(ctx context.Context, workspaceBuil
 	return q.db.GetWorkspaceBuildParameters(ctx, workspaceBuildID)
 }

+func (q *querier) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
+		return nil, err
+	}
+	return q.db.GetWorkspaceBuildStatsByTemplates(ctx, since)
+}
+
 func (q *querier) GetWorkspaceBuildsByWorkspaceID(ctx context.Context, arg database.GetWorkspaceBuildsByWorkspaceIDParams) ([]database.WorkspaceBuild, error) {
 	if _, err := q.GetWorkspaceByID(ctx, arg.WorkspaceID); err != nil {
 		return nil, err
@@ -3966,6 +3987,13 @@ func (q *querier) UpsertLogoURL(ctx context.Context, value string) error {
 	return q.db.UpsertLogoURL(ctx, value)
 }

+func (q *querier) UpsertNotificationReportGeneratorLog(ctx context.Context, arg database.UpsertNotificationReportGeneratorLogParams) error {
+	if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
+		return err
+	}
+	return q.db.UpsertNotificationReportGeneratorLog(ctx, arg)
+}
+
 func (q *querier) UpsertNotificationsSettings(ctx context.Context, value string) error {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil {
 		return err
@@ -2819,6 +2819,28 @@ func (s *MethodTestSuite) TestSystemFunctions() {
 			Value: "value",
 		}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
 	}))
+	s.Run("GetFailedWorkspaceBuildsByTemplateID", s.Subtest(func(db database.Store, check *expects) {
+		check.Args(database.GetFailedWorkspaceBuildsByTemplateIDParams{
+			TemplateID: uuid.New(),
+			Since:      dbtime.Now(),
+		}).Asserts(rbac.ResourceSystem, policy.ActionRead)
+	}))
+	s.Run("GetNotificationReportGeneratorLogByTemplate", s.Subtest(func(db database.Store, check *expects) {
+		_ = db.UpsertNotificationReportGeneratorLog(context.Background(), database.UpsertNotificationReportGeneratorLogParams{
+			NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
+			LastGeneratedAt:        dbtime.Now(),
+		})
+		check.Args(notifications.TemplateWorkspaceBuildsFailedReport).Asserts(rbac.ResourceSystem, policy.ActionRead)
+	}))
+	s.Run("GetWorkspaceBuildStatsByTemplates", s.Subtest(func(db database.Store, check *expects) {
+		check.Args(dbtime.Now()).Asserts(rbac.ResourceSystem, policy.ActionRead)
+	}))
+	s.Run("UpsertNotificationReportGeneratorLog", s.Subtest(func(db database.Store, check *expects) {
+		check.Args(database.UpsertNotificationReportGeneratorLogParams{
+			NotificationTemplateID: uuid.New(),
+			LastGeneratedAt:        dbtime.Now(),
+		}).Asserts(rbac.ResourceSystem, policy.ActionCreate)
+	}))
 }

 func (s *MethodTestSuite) TestNotifications() {
@@ -187,53 +187,54 @@ type data struct {
 	userLinks           []database.UserLink

 	// New tables
-	workspaceAgentStats           []database.WorkspaceAgentStat
-	auditLogs                     []database.AuditLog
-	cryptoKeys                    []database.CryptoKey
-	dbcryptKeys                   []database.DBCryptKey
-	files                         []database.File
-	externalAuthLinks             []database.ExternalAuthLink
-	gitSSHKey                     []database.GitSSHKey
-	groupMembers                  []database.GroupMemberTable
-	groups                        []database.Group
-	jfrogXRayScans                []database.JfrogXrayScan
-	licenses                      []database.License
-	notificationMessages          []database.NotificationMessage
-	notificationPreferences       []database.NotificationPreference
-	oauth2ProviderApps            []database.OAuth2ProviderApp
-	oauth2ProviderAppSecrets      []database.OAuth2ProviderAppSecret
-	oauth2ProviderAppCodes        []database.OAuth2ProviderAppCode
-	oauth2ProviderAppTokens       []database.OAuth2ProviderAppToken
-	parameterSchemas              []database.ParameterSchema
-	provisionerDaemons            []database.ProvisionerDaemon
-	provisionerJobLogs            []database.ProvisionerJobLog
-	provisionerJobs               []database.ProvisionerJob
-	provisionerKeys               []database.ProvisionerKey
-	replicas                      []database.Replica
-	templateVersions              []database.TemplateVersionTable
-	templateVersionParameters     []database.TemplateVersionParameter
-	templateVersionVariables      []database.TemplateVersionVariable
-	templateVersionWorkspaceTags  []database.TemplateVersionWorkspaceTag
-	templates                     []database.TemplateTable
-	templateUsageStats            []database.TemplateUsageStat
-	workspaceAgents               []database.WorkspaceAgent
-	workspaceAgentMetadata        []database.WorkspaceAgentMetadatum
-	workspaceAgentLogs            []database.WorkspaceAgentLog
-	workspaceAgentLogSources      []database.WorkspaceAgentLogSource
-	workspaceAgentScripts         []database.WorkspaceAgentScript
-	workspaceAgentPortShares      []database.WorkspaceAgentPortShare
-	workspaceApps                 []database.WorkspaceApp
-	workspaceAppStatsLastInsertID int64
-	workspaceAppStats             []database.WorkspaceAppStat
-	workspaceBuilds               []database.WorkspaceBuild
-	workspaceBuildParameters      []database.WorkspaceBuildParameter
-	workspaceResourceMetadata     []database.WorkspaceResourceMetadatum
-	workspaceResources            []database.WorkspaceResource
-	workspaces                    []database.Workspace
-	workspaceProxies              []database.WorkspaceProxy
-	customRoles                   []database.CustomRole
-	provisionerJobTimings         []database.ProvisionerJobTiming
-	runtimeConfig                 map[string]string
+	auditLogs                       []database.AuditLog
+	cryptoKeys                      []database.CryptoKey
+	dbcryptKeys                     []database.DBCryptKey
+	files                           []database.File
+	externalAuthLinks               []database.ExternalAuthLink
+	gitSSHKey                       []database.GitSSHKey
+	groupMembers                    []database.GroupMemberTable
+	groups                          []database.Group
+	jfrogXRayScans                  []database.JfrogXrayScan
+	licenses                        []database.License
+	notificationMessages            []database.NotificationMessage
+	notificationPreferences         []database.NotificationPreference
+	notificationReportGeneratorLogs []database.NotificationReportGeneratorLog
+	oauth2ProviderApps              []database.OAuth2ProviderApp
+	oauth2ProviderAppSecrets        []database.OAuth2ProviderAppSecret
+	oauth2ProviderAppCodes          []database.OAuth2ProviderAppCode
+	oauth2ProviderAppTokens         []database.OAuth2ProviderAppToken
+	parameterSchemas                []database.ParameterSchema
+	provisionerDaemons              []database.ProvisionerDaemon
+	provisionerJobLogs              []database.ProvisionerJobLog
+	provisionerJobs                 []database.ProvisionerJob
+	provisionerKeys                 []database.ProvisionerKey
+	replicas                        []database.Replica
+	templateVersions                []database.TemplateVersionTable
+	templateVersionParameters       []database.TemplateVersionParameter
+	templateVersionVariables        []database.TemplateVersionVariable
+	templateVersionWorkspaceTags    []database.TemplateVersionWorkspaceTag
+	templates                       []database.TemplateTable
+	templateUsageStats              []database.TemplateUsageStat
+	workspaceAgents                 []database.WorkspaceAgent
+	workspaceAgentMetadata          []database.WorkspaceAgentMetadatum
+	workspaceAgentLogs              []database.WorkspaceAgentLog
+	workspaceAgentLogSources        []database.WorkspaceAgentLogSource
+	workspaceAgentPortShares        []database.WorkspaceAgentPortShare
+	workspaceAgentScripts           []database.WorkspaceAgentScript
+	workspaceAgentStats             []database.WorkspaceAgentStat
+	workspaceApps                   []database.WorkspaceApp
+	workspaceAppStatsLastInsertID   int64
+	workspaceAppStats               []database.WorkspaceAppStat
+	workspaceBuilds                 []database.WorkspaceBuild
+	workspaceBuildParameters        []database.WorkspaceBuildParameter
+	workspaceResourceMetadata       []database.WorkspaceResourceMetadatum
+	workspaceResources              []database.WorkspaceResource
+	workspaces                      []database.Workspace
+	workspaceProxies                []database.WorkspaceProxy
+	customRoles                     []database.CustomRole
+	provisionerJobTimings           []database.ProvisionerJobTiming
+	runtimeConfig                   map[string]string
 	// Locks is a map of lock names. Any keys within the map are currently
 	// locked.
 	locks                            map[int64]struct{}
@@ -2621,6 +2622,75 @@ func (q *FakeQuerier) GetExternalAuthLinksByUserID(_ context.Context, userID uui
 	return gals, nil
 }

+func (q *FakeQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return nil, err
+	}
+
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	workspaceBuildStats := []database.GetFailedWorkspaceBuildsByTemplateIDRow{}
+	for _, wb := range q.workspaceBuilds {
+		job, err := q.getProvisionerJobByIDNoLock(ctx, wb.JobID)
+		if err != nil {
+			return nil, xerrors.Errorf("get provisioner job by ID: %w", err)
+		}
+
+		if job.JobStatus != database.ProvisionerJobStatusFailed {
+			continue
+		}
+
+		if !job.CompletedAt.Valid {
+			continue
+		}
+
+		if wb.CreatedAt.Before(arg.Since) {
+			continue
+		}
+
+		w, err := q.getWorkspaceByIDNoLock(ctx, wb.WorkspaceID)
+		if err != nil {
+			return nil, xerrors.Errorf("get workspace by ID: %w", err)
+		}
+
+		t, err := q.getTemplateByIDNoLock(ctx, w.TemplateID)
+		if err != nil {
+			return nil, xerrors.Errorf("get template by ID: %w", err)
+		}
+
+		if t.ID != arg.TemplateID {
+			continue
+		}
+
+		workspaceOwner, err := q.getUserByIDNoLock(w.OwnerID)
+		if err != nil {
+			return nil, xerrors.Errorf("get user by ID: %w", err)
+		}
+
+		templateVersion, err := q.getTemplateVersionByIDNoLock(ctx, wb.TemplateVersionID)
+		if err != nil {
+			return nil, xerrors.Errorf("get template version by ID: %w", err)
+		}
+
+		workspaceBuildStats = append(workspaceBuildStats, database.GetFailedWorkspaceBuildsByTemplateIDRow{
+			WorkspaceName:          w.Name,
+			WorkspaceOwnerUsername: workspaceOwner.Username,
+			TemplateVersionName:    templateVersion.Name,
+			WorkspaceBuildNumber:   wb.BuildNumber,
+		})
+	}
+
+	sort.Slice(workspaceBuildStats, func(i, j int) bool {
+		if workspaceBuildStats[i].TemplateVersionName != workspaceBuildStats[j].TemplateVersionName {
+			return workspaceBuildStats[i].TemplateVersionName < workspaceBuildStats[j].TemplateVersionName
+		}
+		return workspaceBuildStats[i].WorkspaceBuildNumber > workspaceBuildStats[j].WorkspaceBuildNumber
+	})
+	return workspaceBuildStats, nil
+}
+
 func (q *FakeQuerier) GetFileByHashAndCreator(_ context.Context, arg database.GetFileByHashAndCreatorParams) (database.File, error) {
 	if err := validateDatabaseType(arg); err != nil {
 		return database.File{}, err
@@ -3044,6 +3114,23 @@ func (q *FakeQuerier) GetNotificationMessagesByStatus(_ context.Context, arg dat
 	return out, nil
 }

+func (q *FakeQuerier) GetNotificationReportGeneratorLogByTemplate(_ context.Context, templateID uuid.UUID) (database.NotificationReportGeneratorLog, error) {
+	err := validateDatabaseType(templateID)
+	if err != nil {
+		return database.NotificationReportGeneratorLog{}, err
+	}
+
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	for _, record := range q.notificationReportGeneratorLogs {
+		if record.NotificationTemplateID == templateID {
+			return record, nil
+		}
+	}
+	return database.NotificationReportGeneratorLog{}, sql.ErrNoRows
+}
+
 func (*FakeQuerier) GetNotificationTemplateByID(_ context.Context, _ uuid.UUID) (database.NotificationTemplate, error) {
 	// Not implementing this function because it relies on state in the database which is created with migrations.
 	// We could consider using code-generation to align the database state and dbmem, but it's not worth it right now.
@@ -5964,6 +6051,63 @@ func (q *FakeQuerier) GetWorkspaceBuildParameters(_ context.Context, workspaceBu
 	return params, nil
 }

+func (q *FakeQuerier) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
+	q.mutex.RLock()
+	defer q.mutex.RUnlock()
+
+	templateStats := map[uuid.UUID]database.GetWorkspaceBuildStatsByTemplatesRow{}
+	for _, wb := range q.workspaceBuilds {
+		job, err := q.getProvisionerJobByIDNoLock(ctx, wb.JobID)
+		if err != nil {
+			return nil, xerrors.Errorf("get provisioner job by ID: %w", err)
+		}
+
+		if !job.CompletedAt.Valid {
+			continue
+		}
+
+		if wb.CreatedAt.Before(since) {
+			continue
+		}
+
+		w, err := q.getWorkspaceByIDNoLock(ctx, wb.WorkspaceID)
+		if err != nil {
+			return nil, xerrors.Errorf("get workspace by ID: %w", err)
+		}
+
+		if _, ok := templateStats[w.TemplateID]; !ok {
+			t, err := q.getTemplateByIDNoLock(ctx, w.TemplateID)
+			if err != nil {
+				return nil, xerrors.Errorf("get template by ID: %w", err)
+			}
+
+			templateStats[w.TemplateID] = database.GetWorkspaceBuildStatsByTemplatesRow{
+				TemplateID:             w.TemplateID,
+				TemplateName:           t.Name,
+				TemplateDisplayName:    t.DisplayName,
+				TemplateOrganizationID: w.OrganizationID,
+			}
+		}
+
+		s := templateStats[w.TemplateID]
+		s.TotalBuilds++
+		if job.JobStatus == database.ProvisionerJobStatusFailed {
+			s.FailedBuilds++
+		}
+		templateStats[w.TemplateID] = s
+	}
+
+	rows := make([]database.GetWorkspaceBuildStatsByTemplatesRow, 0, len(templateStats))
+	for _, ts := range templateStats {
+		rows = append(rows, ts)
+	}
+
+	sort.Slice(rows, func(i, j int) bool {
+		return rows[i].TemplateName < rows[j].TemplateName
+	})
+	return rows, nil
+}
+
 func (q *FakeQuerier) GetWorkspaceBuildsByWorkspaceID(_ context.Context,
 	params database.GetWorkspaceBuildsByWorkspaceIDParams,
 ) ([]database.WorkspaceBuild, error) {
@@ -9440,6 +9584,26 @@ func (q *FakeQuerier) UpsertLogoURL(_ context.Context, data string) error {
 	return nil
 }

+func (q *FakeQuerier) UpsertNotificationReportGeneratorLog(_ context.Context, arg database.UpsertNotificationReportGeneratorLogParams) error {
+	err := validateDatabaseType(arg)
+	if err != nil {
+		return err
+	}
+
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+
+	for i, record := range q.notificationReportGeneratorLogs {
+		if arg.NotificationTemplateID == record.NotificationTemplateID {
+			q.notificationReportGeneratorLogs[i].LastGeneratedAt = arg.LastGeneratedAt
+			return nil
+		}
+	}
+
+	q.notificationReportGeneratorLogs = append(q.notificationReportGeneratorLogs, database.NotificationReportGeneratorLog(arg))
+	return nil
+}
+
 func (q *FakeQuerier) UpsertNotificationsSettings(_ context.Context, data string) error {
 	q.mutex.Lock()
 	defer q.mutex.Unlock()
@@ -634,6 +634,13 @@ func (m metricsStore) GetExternalAuthLinksByUserID(ctx context.Context, userID u
 	return r0, r1
 }

+func (m metricsStore) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetFailedWorkspaceBuildsByTemplateID(ctx, arg)
+	m.queryLatencies.WithLabelValues("GetFailedWorkspaceBuildsByTemplateID").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m metricsStore) GetFileByHashAndCreator(ctx context.Context, arg database.GetFileByHashAndCreatorParams) (database.File, error) {
 	start := time.Now()
 	file, err := m.s.GetFileByHashAndCreator(ctx, arg)
@@ -788,6 +795,13 @@ func (m metricsStore) GetNotificationMessagesByStatus(ctx context.Context, arg d
 	return r0, r1
 }

+func (m metricsStore) GetNotificationReportGeneratorLogByTemplate(ctx context.Context, arg uuid.UUID) (database.NotificationReportGeneratorLog, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetNotificationReportGeneratorLogByTemplate(ctx, arg)
+	m.queryLatencies.WithLabelValues("GetNotificationReportGeneratorLogByTemplate").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m metricsStore) GetNotificationTemplateByID(ctx context.Context, id uuid.UUID) (database.NotificationTemplate, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetNotificationTemplateByID(ctx, id)
@@ -1474,6 +1488,13 @@ func (m metricsStore) GetWorkspaceBuildParameters(ctx context.Context, workspace
 	return params, err
 }

+func (m metricsStore) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetWorkspaceBuildStatsByTemplates(ctx, since)
+	m.queryLatencies.WithLabelValues("GetWorkspaceBuildStatsByTemplates").Observe(time.Since(start).Seconds())
+	return r0, r1
+}
+
 func (m metricsStore) GetWorkspaceBuildsByWorkspaceID(ctx context.Context, arg database.GetWorkspaceBuildsByWorkspaceIDParams) ([]database.WorkspaceBuild, error) {
 	start := time.Now()
 	builds, err := m.s.GetWorkspaceBuildsByWorkspaceID(ctx, arg)
@@ -2517,6 +2538,13 @@ func (m metricsStore) UpsertLogoURL(ctx context.Context, value string) error {
 	return r0
 }

+func (m metricsStore) UpsertNotificationReportGeneratorLog(ctx context.Context, arg database.UpsertNotificationReportGeneratorLogParams) error {
+	start := time.Now()
+	r0 := m.s.UpsertNotificationReportGeneratorLog(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpsertNotificationReportGeneratorLog").Observe(time.Since(start).Seconds())
+	return r0
+}
+
 func (m metricsStore) UpsertNotificationsSettings(ctx context.Context, value string) error {
 	start := time.Now()
 	r0 := m.s.UpsertNotificationsSettings(ctx, value)
@@ -1253,6 +1253,21 @@ func (mr *MockStoreMockRecorder) GetExternalAuthLinksByUserID(arg0, arg1 any) *g
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalAuthLinksByUserID", reflect.TypeOf((*MockStore)(nil).GetExternalAuthLinksByUserID), arg0, arg1)
 }

+// GetFailedWorkspaceBuildsByTemplateID mocks base method.
+func (m *MockStore) GetFailedWorkspaceBuildsByTemplateID(arg0 context.Context, arg1 database.GetFailedWorkspaceBuildsByTemplateIDParams) ([]database.GetFailedWorkspaceBuildsByTemplateIDRow, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetFailedWorkspaceBuildsByTemplateID", arg0, arg1)
+	ret0, _ := ret[0].([]database.GetFailedWorkspaceBuildsByTemplateIDRow)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetFailedWorkspaceBuildsByTemplateID indicates an expected call of GetFailedWorkspaceBuildsByTemplateID.
+func (mr *MockStoreMockRecorder) GetFailedWorkspaceBuildsByTemplateID(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFailedWorkspaceBuildsByTemplateID", reflect.TypeOf((*MockStore)(nil).GetFailedWorkspaceBuildsByTemplateID), arg0, arg1)
+}
+
 // GetFileByHashAndCreator mocks base method.
 func (m *MockStore) GetFileByHashAndCreator(arg0 context.Context, arg1 database.GetFileByHashAndCreatorParams) (database.File, error) {
 	m.ctrl.T.Helper()
@@ -1583,6 +1598,21 @@ func (mr *MockStoreMockRecorder) GetNotificationMessagesByStatus(arg0, arg1 any)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationMessagesByStatus", reflect.TypeOf((*MockStore)(nil).GetNotificationMessagesByStatus), arg0, arg1)
 }

+// GetNotificationReportGeneratorLogByTemplate mocks base method.
+func (m *MockStore) GetNotificationReportGeneratorLogByTemplate(arg0 context.Context, arg1 uuid.UUID) (database.NotificationReportGeneratorLog, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetNotificationReportGeneratorLogByTemplate", arg0, arg1)
+	ret0, _ := ret[0].(database.NotificationReportGeneratorLog)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetNotificationReportGeneratorLogByTemplate indicates an expected call of GetNotificationReportGeneratorLogByTemplate.
+func (mr *MockStoreMockRecorder) GetNotificationReportGeneratorLogByTemplate(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNotificationReportGeneratorLogByTemplate", reflect.TypeOf((*MockStore)(nil).GetNotificationReportGeneratorLogByTemplate), arg0, arg1)
+}
+
 // GetNotificationTemplateByID mocks base method.
 func (m *MockStore) GetNotificationTemplateByID(arg0 context.Context, arg1 uuid.UUID) (database.NotificationTemplate, error) {
 	m.ctrl.T.Helper()
@@ -3083,6 +3113,21 @@ func (mr *MockStoreMockRecorder) GetWorkspaceBuildParameters(arg0, arg1 any) *go
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceBuildParameters", reflect.TypeOf((*MockStore)(nil).GetWorkspaceBuildParameters), arg0, arg1)
 }

+// GetWorkspaceBuildStatsByTemplates mocks base method.
+func (m *MockStore) GetWorkspaceBuildStatsByTemplates(arg0 context.Context, arg1 time.Time) ([]database.GetWorkspaceBuildStatsByTemplatesRow, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetWorkspaceBuildStatsByTemplates", arg0, arg1)
+	ret0, _ := ret[0].([]database.GetWorkspaceBuildStatsByTemplatesRow)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetWorkspaceBuildStatsByTemplates indicates an expected call of GetWorkspaceBuildStatsByTemplates.
+func (mr *MockStoreMockRecorder) GetWorkspaceBuildStatsByTemplates(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWorkspaceBuildStatsByTemplates", reflect.TypeOf((*MockStore)(nil).GetWorkspaceBuildStatsByTemplates), arg0, arg1)
+}
+
 // GetWorkspaceBuildsByWorkspaceID mocks base method.
 func (m *MockStore) GetWorkspaceBuildsByWorkspaceID(arg0 context.Context, arg1 database.GetWorkspaceBuildsByWorkspaceIDParams) ([]database.WorkspaceBuild, error) {
 	m.ctrl.T.Helper()
@@ -5287,6 +5332,20 @@ func (mr *MockStoreMockRecorder) UpsertLogoURL(arg0, arg1 any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertLogoURL", reflect.TypeOf((*MockStore)(nil).UpsertLogoURL), arg0, arg1)
 }

+// UpsertNotificationReportGeneratorLog mocks base method.
+func (m *MockStore) UpsertNotificationReportGeneratorLog(arg0 context.Context, arg1 database.UpsertNotificationReportGeneratorLogParams) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpsertNotificationReportGeneratorLog", arg0, arg1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// UpsertNotificationReportGeneratorLog indicates an expected call of UpsertNotificationReportGeneratorLog.
+func (mr *MockStoreMockRecorder) UpsertNotificationReportGeneratorLog(arg0, arg1 any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertNotificationReportGeneratorLog", reflect.TypeOf((*MockStore)(nil).UpsertNotificationReportGeneratorLog), arg0, arg1)
+}
+
 // UpsertNotificationsSettings mocks base method.
 func (m *MockStore) UpsertNotificationsSettings(arg0 context.Context, arg1 string) error {
 	m.ctrl.T.Helper()
@@ -751,6 +751,13 @@ CREATE TABLE notification_preferences (
    updated_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL
 );

+CREATE TABLE notification_report_generator_logs (
+    notification_template_id uuid NOT NULL,
+    last_generated_at timestamp with time zone NOT NULL
+);
+
+COMMENT ON TABLE notification_report_generator_logs IS 'Log of generated reports for users.';
+
 CREATE TABLE notification_templates (
    id uuid NOT NULL,
    name text NOT NULL,
@@ -1726,6 +1733,9 @@ ALTER TABLE ONLY notification_messages
 ALTER TABLE ONLY notification_preferences
    ADD CONSTRAINT notification_preferences_pkey PRIMARY KEY (user_id, notification_template_id);

+ALTER TABLE ONLY notification_report_generator_logs
+    ADD CONSTRAINT notification_report_generator_logs_pkey PRIMARY KEY (notification_template_id);
+
 ALTER TABLE ONLY notification_templates
    ADD CONSTRAINT notification_templates_name_key UNIQUE (name);

@@ -10,6 +10,7 @@ const (
 	LockIDEnterpriseDeploymentSetup
 	LockIDDBRollup
 	LockIDDBPurge
+	LockIDNotificationsReportGenerator
 )

 // GenLockID generates a unique and consistent lock ID from a given string.
@@ -0,0 +1,3 @@
+DELETE FROM notification_templates WHERE id = '34a20db2-e9cc-4a93-b0e4-8569699d7a00';
+
+DROP TABLE notification_report_generator_logs;
@@ -0,0 +1,30 @@
+INSERT INTO notification_templates (id, name, title_template, body_template, "group", actions)
+VALUES ('34a20db2-e9cc-4a93-b0e4-8569699d7a00', 'Report: Workspace Builds Failed For Template', E'Workspace builds failed for template "{{.Labels.template_display_name}}"',
+        E'Hi {{.UserName}},
+
+Template **{{.Labels.template_display_name}}** has failed to build {{.Data.failed_builds}}/{{.Data.total_builds}} times over the last {{.Data.report_frequency}}.
+
+**Report:**
+{{range $version := .Data.template_versions}}
+**{{$version.template_version_name}}** failed {{$version.failed_count}} time{{if gt $version.failed_count 1}}s{{end}}:
+{{range $build := $version.failed_builds}}
+* [{{$build.workspace_owner_username}} / {{$build.workspace_name}} / #{{$build.build_number}}]({{base_url}}/@{{$build.workspace_owner_username}}/{{$build.workspace_name}}/builds/{{$build.build_number}})
+{{- end}}
+{{end}}
+We recommend reviewing these issues to ensure future builds are successful.',
+        'Template Events', '[
+        {
+            "label": "View workspaces",
+            "url": "{{ base_url }}/workspaces?filter=template%3A{{.Labels.template_name}}"
+        }
+    ]'::jsonb);
+
+CREATE TABLE notification_report_generator_logs
+(
+    notification_template_id uuid NOT NULL,
+    last_generated_at timestamp with time zone NOT NULL,
+
+    PRIMARY KEY (notification_template_id)
+);
+
+COMMENT ON TABLE notification_report_generator_logs IS 'Log of generated reports for users.';
@@ -268,6 +268,7 @@ func TestMigrateUpWithFixtures(t *testing.T) {
 		"template_version_variables",
 		"dbcrypt_keys", // having zero rows is a valid state for this table
 		"template_version_workspace_tags",
+		"notification_report_generator_logs",
 	}
 	s := &tableStats{s: make(map[string]int)}

@@ -2262,6 +2262,12 @@ type NotificationPreference struct {
 	UpdatedAt              time.Time `db:"updated_at" json:"updated_at"`
 }

+// Log of generated reports for users.
+type NotificationReportGeneratorLog struct {
+	NotificationTemplateID uuid.UUID `db:"notification_template_id" json:"notification_template_id"`
+	LastGeneratedAt        time.Time `db:"last_generated_at" json:"last_generated_at"`
+}
+
 // Templates from which to create notification messages.
 type NotificationTemplate struct {
 	ID            uuid.UUID      `db:"id" json:"id"`
@@ -144,6 +144,7 @@ type sqlcQuerier interface {
 	GetDeploymentWorkspaceStats(ctx context.Context) (GetDeploymentWorkspaceStatsRow, error)
 	GetExternalAuthLink(ctx context.Context, arg GetExternalAuthLinkParams) (ExternalAuthLink, error)
 	GetExternalAuthLinksByUserID(ctx context.Context, userID uuid.UUID) ([]ExternalAuthLink, error)
+	GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg GetFailedWorkspaceBuildsByTemplateIDParams) ([]GetFailedWorkspaceBuildsByTemplateIDRow, error)
 	GetFileByHashAndCreator(ctx context.Context, arg GetFileByHashAndCreatorParams) (File, error)
 	GetFileByID(ctx context.Context, id uuid.UUID) (File, error)
 	// Get all templates that use a file.
@@ -170,6 +171,8 @@ type sqlcQuerier interface {
 	GetLicenses(ctx context.Context) ([]License, error)
 	GetLogoURL(ctx context.Context) (string, error)
 	GetNotificationMessagesByStatus(ctx context.Context, arg GetNotificationMessagesByStatusParams) ([]NotificationMessage, error)
+	// Fetch the notification report generator log indicating recent activity.
+	GetNotificationReportGeneratorLogByTemplate(ctx context.Context, templateID uuid.UUID) (NotificationReportGeneratorLog, error)
 	GetNotificationTemplateByID(ctx context.Context, id uuid.UUID) (NotificationTemplate, error)
 	GetNotificationTemplatesByKind(ctx context.Context, kind NotificationTemplateKind) ([]NotificationTemplate, error)
 	GetNotificationsSettings(ctx context.Context) (string, error)
@@ -307,6 +310,7 @@ type sqlcQuerier interface {
 	GetWorkspaceBuildByJobID(ctx context.Context, jobID uuid.UUID) (WorkspaceBuild, error)
 	GetWorkspaceBuildByWorkspaceIDAndBuildNumber(ctx context.Context, arg GetWorkspaceBuildByWorkspaceIDAndBuildNumberParams) (WorkspaceBuild, error)
 	GetWorkspaceBuildParameters(ctx context.Context, workspaceBuildID uuid.UUID) ([]WorkspaceBuildParameter, error)
+	GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]GetWorkspaceBuildStatsByTemplatesRow, error)
 	GetWorkspaceBuildsByWorkspaceID(ctx context.Context, arg GetWorkspaceBuildsByWorkspaceIDParams) ([]WorkspaceBuild, error)
 	GetWorkspaceBuildsCreatedAfter(ctx context.Context, createdAt time.Time) ([]WorkspaceBuild, error)
 	GetWorkspaceByAgentID(ctx context.Context, agentID uuid.UUID) (GetWorkspaceByAgentIDRow, error)
@@ -489,6 +493,8 @@ type sqlcQuerier interface {
 	UpsertJFrogXrayScanByWorkspaceAndAgentID(ctx context.Context, arg UpsertJFrogXrayScanByWorkspaceAndAgentIDParams) error
 	UpsertLastUpdateCheck(ctx context.Context, value string) error
 	UpsertLogoURL(ctx context.Context, value string) error
+	// Insert or update notification report generator logs with recent activity.
+	UpsertNotificationReportGeneratorLog(ctx context.Context, arg UpsertNotificationReportGeneratorLogParams) error
 	UpsertNotificationsSettings(ctx context.Context, value string) error
 	UpsertOAuthSigningKey(ctx context.Context, value string) error
 	UpsertProvisionerDaemon(ctx context.Context, arg UpsertProvisionerDaemonParams) (ProvisionerDaemon, error)
@@ -3879,6 +3879,23 @@ func (q *sqlQuerier) GetNotificationMessagesByStatus(ctx context.Context, arg Ge
 	return items, nil
 }

+const getNotificationReportGeneratorLogByTemplate = `-- name: GetNotificationReportGeneratorLogByTemplate :one
+SELECT
+	notification_template_id, last_generated_at
+FROM
+	notification_report_generator_logs
+WHERE
+	notification_template_id = $1::uuid
+`
+
+// Fetch the notification report generator log indicating recent activity.
+func (q *sqlQuerier) GetNotificationReportGeneratorLogByTemplate(ctx context.Context, templateID uuid.UUID) (NotificationReportGeneratorLog, error) {
+	row := q.db.QueryRowContext(ctx, getNotificationReportGeneratorLogByTemplate, templateID)
+	var i NotificationReportGeneratorLog
+	err := row.Scan(&i.NotificationTemplateID, &i.LastGeneratedAt)
+	return i, err
+}
+
 const getNotificationTemplateByID = `-- name: GetNotificationTemplateByID :one
 SELECT id, name, title_template, body_template, actions, "group", method, kind
 FROM notification_templates
@@ -4028,6 +4045,23 @@ func (q *sqlQuerier) UpdateUserNotificationPreferences(ctx context.Context, arg
 	return result.RowsAffected()
 }

+const upsertNotificationReportGeneratorLog = `-- name: UpsertNotificationReportGeneratorLog :exec
+INSERT INTO notification_report_generator_logs (notification_template_id, last_generated_at) VALUES ($1, $2)
+ON CONFLICT (notification_template_id) DO UPDATE set last_generated_at = EXCLUDED.last_generated_at
+WHERE notification_report_generator_logs.notification_template_id = EXCLUDED.notification_template_id
+`
+
+type UpsertNotificationReportGeneratorLogParams struct {
+	NotificationTemplateID uuid.UUID `db:"notification_template_id" json:"notification_template_id"`
+	LastGeneratedAt        time.Time `db:"last_generated_at" json:"last_generated_at"`
+}
+
+// Insert or update notification report generator logs with recent activity.
+func (q *sqlQuerier) UpsertNotificationReportGeneratorLog(ctx context.Context, arg UpsertNotificationReportGeneratorLogParams) error {
+	_, err := q.db.ExecContext(ctx, upsertNotificationReportGeneratorLog, arg.NotificationTemplateID, arg.LastGeneratedAt)
+	return err
+}
+
 const deleteOAuth2ProviderAppByID = `-- name: DeleteOAuth2ProviderAppByID :exec
 DELETE FROM oauth2_provider_apps WHERE id = $1
 `
@@ -12896,6 +12930,83 @@ func (q *sqlQuerier) GetActiveWorkspaceBuildsByTemplateID(ctx context.Context, t
 	return items, nil
 }

+const getFailedWorkspaceBuildsByTemplateID = `-- name: GetFailedWorkspaceBuildsByTemplateID :many
+SELECT
+	tv.name AS template_version_name,
+	u.username AS workspace_owner_username,
+	w.name AS workspace_name,
+	wb.build_number AS workspace_build_number
+FROM
+	workspace_build_with_user AS wb
+JOIN
+	workspaces AS w
+ON
+	wb.workspace_id = w.id
+JOIN
+    users AS u
+ON
+    w.owner_id = u.id
+JOIN
+	provisioner_jobs AS pj
+ON
+	wb.job_id = pj.id
+JOIN
+	templates AS t
+ON
+	w.template_id = t.id
+JOIN
+	template_versions AS tv
+ON
+	wb.template_version_id = tv.id
+WHERE
+	w.template_id = $1
+	AND wb.created_at >= $2
+	AND pj.completed_at IS NOT NULL
+	AND pj.job_status = 'failed'
+ORDER BY
+	tv.name ASC, wb.build_number DESC
+`
+
+type GetFailedWorkspaceBuildsByTemplateIDParams struct {
+	TemplateID uuid.UUID `db:"template_id" json:"template_id"`
+	Since      time.Time `db:"since" json:"since"`
+}
+
+type GetFailedWorkspaceBuildsByTemplateIDRow struct {
+	TemplateVersionName    string `db:"template_version_name" json:"template_version_name"`
+	WorkspaceOwnerUsername string `db:"workspace_owner_username" json:"workspace_owner_username"`
+	WorkspaceName          string `db:"workspace_name" json:"workspace_name"`
+	WorkspaceBuildNumber   int32  `db:"workspace_build_number" json:"workspace_build_number"`
+}
+
+func (q *sqlQuerier) GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg GetFailedWorkspaceBuildsByTemplateIDParams) ([]GetFailedWorkspaceBuildsByTemplateIDRow, error) {
+	rows, err := q.db.QueryContext(ctx, getFailedWorkspaceBuildsByTemplateID, arg.TemplateID, arg.Since)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetFailedWorkspaceBuildsByTemplateIDRow
+	for rows.Next() {
+		var i GetFailedWorkspaceBuildsByTemplateIDRow
+		if err := rows.Scan(
+			&i.TemplateVersionName,
+			&i.WorkspaceOwnerUsername,
+			&i.WorkspaceName,
+			&i.WorkspaceBuildNumber,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getLatestWorkspaceBuildByWorkspaceID = `-- name: GetLatestWorkspaceBuildByWorkspaceID :one
 SELECT
 	id, created_at, updated_at, workspace_id, template_version_id, build_number, transition, initiator_id, provisioner_state, job_id, deadline, reason, daily_cost, max_deadline, initiator_by_avatar_url, initiator_by_username
@@ -13154,6 +13265,73 @@ func (q *sqlQuerier) GetWorkspaceBuildByWorkspaceIDAndBuildNumber(ctx context.Co
 	return i, err
 }

+const getWorkspaceBuildStatsByTemplates = `-- name: GetWorkspaceBuildStatsByTemplates :many
+SELECT
+    w.template_id,
+	t.name AS template_name,
+	t.display_name AS template_display_name,
+	t.organization_id AS template_organization_id,
+    COUNT(*) AS total_builds,
+    COUNT(CASE WHEN pj.job_status = 'failed' THEN 1 END) AS failed_builds
+FROM
+    workspace_build_with_user AS wb
+JOIN
+    workspaces AS w ON
+    wb.workspace_id = w.id
+JOIN
+    provisioner_jobs AS pj ON
+    wb.job_id = pj.id
+JOIN
+    templates AS t ON
+	w.template_id = t.id
+WHERE
+    wb.created_at >= $1
+    AND pj.completed_at IS NOT NULL
+GROUP BY
+    w.template_id, template_name, template_display_name, template_organization_id
+ORDER BY
+    template_name ASC
+`
+
+type GetWorkspaceBuildStatsByTemplatesRow struct {
+	TemplateID             uuid.UUID `db:"template_id" json:"template_id"`
+	TemplateName           string    `db:"template_name" json:"template_name"`
+	TemplateDisplayName    string    `db:"template_display_name" json:"template_display_name"`
+	TemplateOrganizationID uuid.UUID `db:"template_organization_id" json:"template_organization_id"`
+	TotalBuilds            int64     `db:"total_builds" json:"total_builds"`
+	FailedBuilds           int64     `db:"failed_builds" json:"failed_builds"`
+}
+
+func (q *sqlQuerier) GetWorkspaceBuildStatsByTemplates(ctx context.Context, since time.Time) ([]GetWorkspaceBuildStatsByTemplatesRow, error) {
+	rows, err := q.db.QueryContext(ctx, getWorkspaceBuildStatsByTemplates, since)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetWorkspaceBuildStatsByTemplatesRow
+	for rows.Next() {
+		var i GetWorkspaceBuildStatsByTemplatesRow
+		if err := rows.Scan(
+			&i.TemplateID,
+			&i.TemplateName,
+			&i.TemplateDisplayName,
+			&i.TemplateOrganizationID,
+			&i.TotalBuilds,
+			&i.FailedBuilds,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getWorkspaceBuildsByWorkspaceID = `-- name: GetWorkspaceBuildsByWorkspaceID :many
 SELECT
 	id, created_at, updated_at, workspace_id, template_version_id, build_number, transition, initiator_id, provisioner_state, job_id, deadline, reason, daily_cost, max_deadline, initiator_by_avatar_url, initiator_by_username
@@ -174,3 +174,18 @@ SELECT *
 FROM notification_templates
 WHERE kind = @kind::notification_template_kind
 ORDER BY name ASC;
+
+-- name: GetNotificationReportGeneratorLogByTemplate :one
+-- Fetch the notification report generator log indicating recent activity.
+SELECT
+	*
+FROM
+	notification_report_generator_logs
+WHERE
+	notification_template_id = @template_id::uuid;
+
+-- name: UpsertNotificationReportGeneratorLog :exec
+-- Insert or update notification report generator logs with recent activity.
+INSERT INTO notification_report_generator_logs (notification_template_id, last_generated_at) VALUES (@notification_template_id, @last_generated_at)
+ON CONFLICT (notification_template_id) DO UPDATE set last_generated_at = EXCLUDED.last_generated_at
+WHERE notification_report_generator_logs.notification_template_id = EXCLUDED.notification_template_id;
@@ -179,3 +179,66 @@ WHERE
 	wb.transition = 'start'::workspace_transition
 AND
 	pj.completed_at IS NOT NULL;
+
+-- name: GetWorkspaceBuildStatsByTemplates :many
+SELECT
+    w.template_id,
+	t.name AS template_name,
+	t.display_name AS template_display_name,
+	t.organization_id AS template_organization_id,
+    COUNT(*) AS total_builds,
+    COUNT(CASE WHEN pj.job_status = 'failed' THEN 1 END) AS failed_builds
+FROM
+    workspace_build_with_user AS wb
+JOIN
+    workspaces AS w ON
+    wb.workspace_id = w.id
+JOIN
+    provisioner_jobs AS pj ON
+    wb.job_id = pj.id
+JOIN
+    templates AS t ON
+	w.template_id = t.id
+WHERE
+    wb.created_at >= @since
+    AND pj.completed_at IS NOT NULL
+GROUP BY
+    w.template_id, template_name, template_display_name, template_organization_id
+ORDER BY
+    template_name ASC;
+
+-- name: GetFailedWorkspaceBuildsByTemplateID :many
+SELECT
+	tv.name AS template_version_name,
+	u.username AS workspace_owner_username,
+	w.name AS workspace_name,
+	wb.build_number AS workspace_build_number
+FROM
+	workspace_build_with_user AS wb
+JOIN
+	workspaces AS w
+ON
+	wb.workspace_id = w.id
+JOIN
+    users AS u
+ON
+    w.owner_id = u.id
+JOIN
+	provisioner_jobs AS pj
+ON
+	wb.job_id = pj.id
+JOIN
+	templates AS t
+ON
+	w.template_id = t.id
+JOIN
+	template_versions AS tv
+ON
+	wb.template_version_id = tv.id
+WHERE
+	w.template_id = $1
+	AND wb.created_at >= @since
+	AND pj.completed_at IS NOT NULL
+	AND pj.job_status = 'failed'
+ORDER BY
+	tv.name ASC, wb.build_number DESC;
@@ -26,6 +26,7 @@ const (
 	UniqueLicensesPkey                                        UniqueConstraint = "licenses_pkey"                                               // ALTER TABLE ONLY licenses ADD CONSTRAINT licenses_pkey PRIMARY KEY (id);
 	UniqueNotificationMessagesPkey                            UniqueConstraint = "notification_messages_pkey"                                  // ALTER TABLE ONLY notification_messages ADD CONSTRAINT notification_messages_pkey PRIMARY KEY (id);
 	UniqueNotificationPreferencesPkey                         UniqueConstraint = "notification_preferences_pkey"                               // ALTER TABLE ONLY notification_preferences ADD CONSTRAINT notification_preferences_pkey PRIMARY KEY (user_id, notification_template_id);
+	UniqueNotificationReportGeneratorLogsPkey                 UniqueConstraint = "notification_report_generator_logs_pkey"                     // ALTER TABLE ONLY notification_report_generator_logs ADD CONSTRAINT notification_report_generator_logs_pkey PRIMARY KEY (notification_template_id);
 	UniqueNotificationTemplatesNameKey                        UniqueConstraint = "notification_templates_name_key"                             // ALTER TABLE ONLY notification_templates ADD CONSTRAINT notification_templates_name_key UNIQUE (name);
 	UniqueNotificationTemplatesPkey                           UniqueConstraint = "notification_templates_pkey"                                 // ALTER TABLE ONLY notification_templates ADD CONSTRAINT notification_templates_pkey PRIMARY KEY (id);
 	UniqueOauth2ProviderAppCodesPkey                          UniqueConstraint = "oauth2_provider_app_codes_pkey"                              // ALTER TABLE ONLY oauth2_provider_app_codes ADD CONSTRAINT oauth2_provider_app_codes_pkey PRIMARY KEY (id);
@@ -52,9 +52,14 @@ func NewStoreEnqueuer(cfg codersdk.NotificationsConfig, store Store, helpers tem
 	}, nil
 }

+// Enqueue queues a notification message for later delivery, assumes no structured input data.
+func (s *StoreEnqueuer) Enqueue(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
+	return s.EnqueueWithData(ctx, userID, templateID, labels, nil, createdBy, targets...)
+}
+
 // Enqueue queues a notification message for later delivery.
 // Messages will be dequeued by a notifier later and dispatched.
-func (s *StoreEnqueuer) Enqueue(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
+func (s *StoreEnqueuer) EnqueueWithData(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, data map[string]any, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
 	metadata, err := s.store.FetchNewMessageMetadata(ctx, database.FetchNewMessageMetadataParams{
 		UserID:                 userID,
 		NotificationTemplateID: templateID,
@@ -69,7 +74,7 @@ func (s *StoreEnqueuer) Enqueue(ctx context.Context, userID, templateID uuid.UUI
 		dispatchMethod = metadata.CustomMethod.NotificationMethod
 	}

-	payload, err := s.buildPayload(metadata, labels)
+	payload, err := s.buildPayload(metadata, labels, data)
 	if err != nil {
 		s.log.Warn(ctx, "failed to build payload", slog.F("template_id", templateID), slog.F("user_id", userID), slog.Error(err))
 		return nil, xerrors.Errorf("enqueue notification (payload build): %w", err)
@@ -119,7 +124,7 @@ func (s *StoreEnqueuer) Enqueue(ctx context.Context, userID, templateID uuid.UUI
 // buildPayload creates the payload that the notification will for variable substitution and/or routing.
 // The payload contains information about the recipient, the event that triggered the notification, and any subsequent
 // actions which can be taken by the recipient.
-func (s *StoreEnqueuer) buildPayload(metadata database.FetchNewMessageMetadataRow, labels map[string]string) (*types.MessagePayload, error) {
+func (s *StoreEnqueuer) buildPayload(metadata database.FetchNewMessageMetadataRow, labels map[string]string, data map[string]any) (*types.MessagePayload, error) {
 	payload := types.MessagePayload{
 		Version: "1.1",

@@ -132,6 +137,8 @@ func (s *StoreEnqueuer) buildPayload(metadata database.FetchNewMessageMetadataRo
 		UserUsername: metadata.UserUsername,

 		Labels: labels,
+		Data:   data,
+
 		// No actions yet
 	}

@@ -162,3 +169,8 @@ func (*NoopEnqueuer) Enqueue(context.Context, uuid.UUID, uuid.UUID, map[string]s
 	// nolint:nilnil // irrelevant.
 	return nil, nil
 }
+
+func (*NoopEnqueuer) EnqueueWithData(context.Context, uuid.UUID, uuid.UUID, map[string]string, map[string]any, string, ...uuid.UUID) (*uuid.UUID, error) {
+	// nolint:nilnil // irrelevant.
+	return nil, nil
+}
@@ -29,4 +29,6 @@ var (
 // Template-related events.
 var (
 	TemplateTemplateDeleted = uuid.MustParse("29a09665-2a4c-403f-9648-54301670e7be")
+
+	TemplateWorkspaceBuildsFailedReport = uuid.MustParse("34a20db2-e9cc-4a93-b0e4-8569699d7a00")
 )
@@ -46,6 +46,7 @@ type Manager struct {
 	notifier *notifier
 	handlers map[database.NotificationMethod]Handler
 	method   database.NotificationMethod
+	helpers  template.FuncMap

 	metrics *Metrics

@@ -108,6 +109,7 @@ func NewManager(cfg codersdk.NotificationsConfig, store Store, helpers template.
 		done: make(chan any),

 		handlers: defaultHandlers(cfg, helpers, log),
+		helpers:  helpers,

 		clock: quartz.NewReal(),
 	}
@@ -169,7 +171,7 @@ func (m *Manager) loop(ctx context.Context) error {
 	var eg errgroup.Group

 	// Create a notifier to run concurrently, which will handle dequeueing and dispatching notifications.
-	m.notifier = newNotifier(m.cfg, uuid.New(), m.log, m.store, m.handlers, m.metrics, m.clock)
+	m.notifier = newNotifier(m.cfg, uuid.New(), m.log, m.store, m.handlers, m.helpers, m.metrics, m.clock)
 	eg.Go(func() error {
 		return m.notifier.run(ctx, m.success, m.failure)
 	})
@@ -844,6 +844,56 @@ func TestNotificationTemplatesCanRender(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "TemplateWorkspaceBuildsFailedReport",
+			id:   notifications.TemplateWorkspaceBuildsFailedReport,
+			payload: types.MessagePayload{
+				UserName: "Bobby",
+				Labels: map[string]string{
+					"template_name":         "bobby-first-template",
+					"template_display_name": "Bobby First Template",
+				},
+				Data: map[string]any{
+					"failed_builds":    4,
+					"total_builds":     55,
+					"report_frequency": "week",
+					"template_versions": []map[string]any{
+						{
+							"template_version_name": "bobby-template-version-1",
+							"failed_count":          3,
+							"failed_builds": []map[string]any{
+								{
+									"workspace_owner_username": "mtojek",
+									"workspace_name":           "workspace-1",
+									"build_number":             1234,
+								},
+								{
+									"workspace_owner_username": "johndoe",
+									"workspace_name":           "my-workspace-3",
+									"build_number":             5678,
+								},
+								{
+									"workspace_owner_username": "jack",
+									"workspace_name":           "workwork",
+									"build_number":             774,
+								},
+							},
+						},
+						{
+							"template_version_name": "bobby-template-version-2",
+							"failed_count":          1,
+							"failed_builds": []map[string]any{
+								{
+									"workspace_owner_username": "ben",
+									"workspace_name":           "cool-workspace",
+									"build_number":             8888,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}

 	allTemplates, err := enumerateAllTemplates(t)
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"sync"
+	"text/template"

 	"github.com/google/uuid"
 	"golang.org/x/sync/errgroup"
@@ -36,13 +37,14 @@ type notifier struct {

 	handlers map[database.NotificationMethod]Handler
 	metrics  *Metrics
+	helpers  template.FuncMap

 	// clock is for testing
 	clock quartz.Clock
 }

 func newNotifier(cfg codersdk.NotificationsConfig, id uuid.UUID, log slog.Logger, db Store,
-	hr map[database.NotificationMethod]Handler, metrics *Metrics, clock quartz.Clock,
+	hr map[database.NotificationMethod]Handler, helpers template.FuncMap, metrics *Metrics, clock quartz.Clock,
 ) *notifier {
 	tick := clock.NewTicker(cfg.FetchInterval.Value(), "notifier", "fetchInterval")
 	return &notifier{
@@ -54,6 +56,7 @@ func newNotifier(cfg codersdk.NotificationsConfig, id uuid.UUID, log slog.Logger
 		tick:     tick,
 		store:    db,
 		handlers: hr,
+		helpers:  helpers,
 		metrics:  metrics,
 		clock:    clock,
 	}
@@ -221,10 +224,10 @@ func (n *notifier) prepare(ctx context.Context, msg database.AcquireNotification
 	}

 	var title, body string
-	if title, err = render.GoTemplate(msg.TitleTemplate, payload, nil); err != nil {
+	if title, err = render.GoTemplate(msg.TitleTemplate, payload, n.helpers); err != nil {
 		return nil, xerrors.Errorf("render title: %w", err)
 	}
-	if body, err = render.GoTemplate(msg.BodyTemplate, payload, nil); err != nil {
+	if body, err = render.GoTemplate(msg.BodyTemplate, payload, n.helpers); err != nil {
 		return nil, xerrors.Errorf("render body: %w", err)
 	}

@@ -0,0 +1,300 @@
+package reports
+
+import (
+	"context"
+	"database/sql"
+	"io"
+	"slices"
+	"sort"
+	"time"
+
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+	"github.com/coder/quartz"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/coderd/notifications"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+const (
+	delay = 15 * time.Minute
+)
+
+func NewReportGenerator(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) io.Closer {
+	closed := make(chan struct{})
+
+	ctx, cancelFunc := context.WithCancel(ctx)
+	//nolint:gocritic // The system generates periodic reports without direct user input.
+	ctx = dbauthz.AsSystemRestricted(ctx)
+
+	// Start the ticker with the initial delay.
+	ticker := clk.NewTicker(delay)
+	ticker.Stop()
+	doTick := func(start time.Time) {
+		defer ticker.Reset(delay)
+		// Start a transaction to grab advisory lock, we don't want to run generator jobs at the same time (multiple replicas).
+		if err := db.InTx(func(tx database.Store) error {
+			// Acquire a lock to ensure that only one instance of the generator is running at a time.
+			ok, err := tx.TryAcquireLock(ctx, database.LockIDNotificationsReportGenerator)
+			if err != nil {
+				return xerrors.Errorf("failed to acquire report generator lock: %w", err)
+			}
+			if !ok {
+				logger.Debug(ctx, "unable to acquire lock for generating periodic reports, skipping")
+				return nil
+			}
+
+			err = reportFailedWorkspaceBuilds(ctx, logger, db, enqueuer, clk)
+			if err != nil {
+				return xerrors.Errorf("unable to generate reports with failed workspace builds: %w", err)
+			}
+
+			logger.Info(ctx, "report generator finished", slog.F("duration", clk.Since(start)))
+
+			return nil
+		}, nil); err != nil {
+			logger.Error(ctx, "failed to generate reports", slog.Error(err))
+			return
+		}
+	}
+
+	go func() {
+		defer close(closed)
+		defer ticker.Stop()
+		// Force an initial tick.
+		doTick(dbtime.Time(clk.Now()).UTC())
+		for {
+			select {
+			case <-ctx.Done():
+				logger.Debug(ctx, "closing report generator")
+				return
+			case tick := <-ticker.C:
+				ticker.Stop()
+				doTick(dbtime.Time(tick).UTC())
+			}
+		}
+	}()
+	return &reportGenerator{
+		cancel: cancelFunc,
+		closed: closed,
+	}
+}
+
+type reportGenerator struct {
+	cancel context.CancelFunc
+	closed chan struct{}
+}
+
+func (i *reportGenerator) Close() error {
+	i.cancel()
+	<-i.closed
+	return nil
+}
+
+const (
+	failedWorkspaceBuildsReportFrequency      = 7 * 24 * time.Hour
+	failedWorkspaceBuildsReportFrequencyLabel = "week"
+)
+
+func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) error {
+	now := clk.Now()
+	since := now.Add(-failedWorkspaceBuildsReportFrequency)
+
+	// Firstly, check if this is the first run of the job ever
+	reportLog, err := db.GetNotificationReportGeneratorLogByTemplate(ctx, notifications.TemplateWorkspaceBuildsFailedReport)
+	if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
+		return xerrors.Errorf("unable to read report generator log: %w", err)
+	}
+	if xerrors.Is(err, sql.ErrNoRows) {
+		// First run? Check-in the job, and get back after one week.
+		logger.Info(ctx, "report generator is executing the job for the first time", slog.F("notification_template_id", notifications.TemplateWorkspaceBuildsFailedReport))
+
+		err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
+			NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
+			LastGeneratedAt:        dbtime.Time(now).UTC(),
+		})
+		if err != nil {
+			return xerrors.Errorf("unable to update report generator logs (first time execution): %w", err)
+		}
+		return nil
+	}
+
+	// Secondly, check if the job has not been running recently
+	if !reportLog.LastGeneratedAt.IsZero() && reportLog.LastGeneratedAt.Add(failedWorkspaceBuildsReportFrequency).After(now) {
+		return nil // reports sent recently, no need to send them now
+	}
+
+	// Thirdly, fetch workspace build stats by templates
+	templateStatsRows, err := db.GetWorkspaceBuildStatsByTemplates(ctx, dbtime.Time(since).UTC())
+	if err != nil {
+		return xerrors.Errorf("unable to fetch failed workspace builds: %w", err)
+	}
+
+	for _, stats := range templateStatsRows {
+		select {
+		case <-ctx.Done():
+			logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
+			break
+		default:
+		}
+
+		if stats.FailedBuilds == 0 {
+			logger.Info(ctx, "no failed workspace builds found for template", slog.F("template_id", stats.TemplateID), slog.Error(err))
+			continue
+		}
+
+		// Fetch template admins with org access to the templates
+		templateAdmins, err := findTemplateAdmins(ctx, db, stats)
+		if err != nil {
+			logger.Error(ctx, "unable to find template admins for template", slog.F("template_id", stats.TemplateID), slog.Error(err))
+			continue
+		}
+
+		// Fetch failed builds by the template
+		failedBuilds, err := db.GetFailedWorkspaceBuildsByTemplateID(ctx, database.GetFailedWorkspaceBuildsByTemplateIDParams{
+			TemplateID: stats.TemplateID,
+			Since:      dbtime.Time(since).UTC(),
+		})
+		if err != nil {
+			logger.Error(ctx, "unable to fetch failed workspace builds", slog.F("template_id", stats.TemplateID), slog.Error(err))
+			continue
+		}
+		reportData := buildDataForReportFailedWorkspaceBuilds(stats, failedBuilds)
+
+		// Send reports to template admins
+		templateDisplayName := stats.TemplateDisplayName
+		if templateDisplayName == "" {
+			templateDisplayName = stats.TemplateName
+		}
+
+		for _, templateAdmin := range templateAdmins {
+			select {
+			case <-ctx.Done():
+				logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
+				break
+			default:
+			}
+
+			if _, err := enqueuer.EnqueueWithData(ctx, templateAdmin.ID, notifications.TemplateWorkspaceBuildsFailedReport,
+				map[string]string{
+					"template_name":         stats.TemplateName,
+					"template_display_name": templateDisplayName,
+				},
+				reportData,
+				"report_generator",
+				stats.TemplateID, stats.TemplateOrganizationID,
+			); err != nil {
+				logger.Warn(ctx, "failed to send a report with failed workspace builds", slog.Error(err))
+			}
+		}
+	}
+
+	if xerrors.Is(ctx.Err(), context.Canceled) {
+		logger.Error(ctx, "report generator job is canceled")
+		return ctx.Err()
+	}
+
+	// Lastly, update the timestamp in the generator log.
+	err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
+		NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
+		LastGeneratedAt:        dbtime.Time(now).UTC(),
+	})
+	if err != nil {
+		return xerrors.Errorf("unable to update report generator logs: %w", err)
+	}
+	return nil
+}
+
+const workspaceBuildsLimitPerTemplateVersion = 10
+
+func buildDataForReportFailedWorkspaceBuilds(stats database.GetWorkspaceBuildStatsByTemplatesRow, failedBuilds []database.GetFailedWorkspaceBuildsByTemplateIDRow) map[string]any {
+	// Build notification model for template versions and failed workspace builds.
+	//
+	// Failed builds are sorted by template version ascending, workspace build number descending.
+	// Review builds, group them by template versions, and assign to builds to template versions.
+	// The map requires `[]map[string]any{}` to be compatible with data passed to `NotificationEnqueuer`.
+	templateVersions := []map[string]any{}
+	for _, failedBuild := range failedBuilds {
+		c := len(templateVersions)
+
+		if c == 0 || templateVersions[c-1]["template_version_name"] != failedBuild.TemplateVersionName {
+			templateVersions = append(templateVersions, map[string]any{
+				"template_version_name": failedBuild.TemplateVersionName,
+				"failed_count":          1,
+				"failed_builds": []map[string]any{
+					{
+						"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
+						"workspace_name":           failedBuild.WorkspaceName,
+						"build_number":             failedBuild.WorkspaceBuildNumber,
+					},
+				},
+			})
+			continue
+		}
+
+		tv := templateVersions[c-1]
+		//nolint:errorlint,forcetypeassert // only this function prepares the notification model
+		tv["failed_count"] = tv["failed_count"].(int) + 1
+
+		//nolint:errorlint,forcetypeassert // only this function prepares the notification model
+		builds := tv["failed_builds"].([]map[string]any)
+		if len(builds) < workspaceBuildsLimitPerTemplateVersion {
+			// return N last builds to prevent long email reports
+			builds = append(builds, map[string]any{
+				"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
+				"workspace_name":           failedBuild.WorkspaceName,
+				"build_number":             failedBuild.WorkspaceBuildNumber,
+			})
+			tv["failed_builds"] = builds
+		}
+		templateVersions[c-1] = tv
+	}
+
+	return map[string]any{
+		"failed_builds":     stats.FailedBuilds,
+		"total_builds":      stats.TotalBuilds,
+		"report_frequency":  failedWorkspaceBuildsReportFrequencyLabel,
+		"template_versions": templateVersions,
+	}
+}
+
+func findTemplateAdmins(ctx context.Context, db database.Store, stats database.GetWorkspaceBuildStatsByTemplatesRow) ([]database.GetUsersRow, error) {
+	users, err := db.GetUsers(ctx, database.GetUsersParams{
+		RbacRole: []string{codersdk.RoleTemplateAdmin},
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("unable to fetch template admins: %w", err)
+	}
+
+	var templateAdmins []database.GetUsersRow
+	if len(users) == 0 {
+		return templateAdmins, nil
+	}
+
+	usersByIDs := map[uuid.UUID]database.GetUsersRow{}
+	var userIDs []uuid.UUID
+	for _, user := range users {
+		usersByIDs[user.ID] = user
+		userIDs = append(userIDs, user.ID)
+	}
+
+	orgIDsByMemberIDs, err := db.GetOrganizationIDsByMemberIDs(ctx, userIDs)
+	if err != nil {
+		return nil, xerrors.Errorf("unable to fetch organization IDs by member IDs: %w", err)
+	}
+
+	for _, entry := range orgIDsByMemberIDs {
+		if slices.Contains(entry.OrganizationIDs, stats.TemplateOrganizationID) {
+			templateAdmins = append(templateAdmins, usersByIDs[entry.UserID])
+		}
+	}
+	sort.Slice(templateAdmins, func(i, j int) bool {
+		return templateAdmins[i].Username < templateAdmins[j].Username
+	})
+	return templateAdmins, nil
+}
@@ -0,0 +1,475 @@
+package reports
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/quartz"
+
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/database/pubsub"
+	"github.com/coder/coder/v2/coderd/notifications"
+	"github.com/coder/coder/v2/coderd/rbac"
+	"github.com/coder/coder/v2/testutil"
+)
+
+const dayDuration = 24 * time.Hour
+
+var (
+	jobError     = sql.NullString{String: "badness", Valid: true}
+	jobErrorCode = sql.NullString{String: "ERR-42", Valid: true}
+)
+
+func TestReportFailedWorkspaceBuilds(t *testing.T) {
+	t.Parallel()
+
+	t.Run("EmptyState_NoBuilds_NoReport", func(t *testing.T) {
+		t.Parallel()
+
+		// Setup
+		ctx, logger, db, _, notifEnq, clk := setup(t)
+
+		// Database is ready, so we can clear notifications queue
+		notifEnq.Clear()
+
+		// When: first run
+		err := reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then: no report should be generated
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent)
+
+		// Given: one week later and no jobs were executed
+		clk.Advance(failedWorkspaceBuildsReportFrequency + time.Minute)
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then: report is still empty
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent)
+	})
+
+	t.Run("InitialState_NoBuilds_NoReport", func(t *testing.T) {
+		t.Parallel()
+
+		// Setup
+		ctx, logger, db, ps, notifEnq, clk := setup(t)
+		now := clk.Now()
+
+		// Organization
+		org := dbgen.Organization(t, db, database.Organization{})
+
+		// Template admins
+		templateAdmin1 := dbgen.User(t, db, database.User{Username: "template-admin-1", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Regular users
+		user1 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user1.ID, OrganizationID: org.ID})
+		user2 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user2.ID, OrganizationID: org.ID})
+
+		// Templates
+		t1 := dbgen.Template(t, db, database.Template{Name: "template-1", DisplayName: "First Template", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Template versions
+		t1v1 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-1", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+
+		// Workspaces
+		w1 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
+
+		w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		// When: first run
+		notifEnq.Clear()
+		err := reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then: failed builds should not be reported
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent)
+
+		// Given: one week later, but still no jobs
+		clk.Advance(failedWorkspaceBuildsReportFrequency + time.Minute)
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then: report is still empty
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent)
+	})
+
+	t.Run("FailedBuilds_SecondRun_Report_ThirdRunTooEarly_NoReport_FourthRun_Report", func(t *testing.T) {
+		t.Parallel()
+
+		verifyNotification := func(t *testing.T, recipient database.User, notif *testutil.Notification, tmpl database.Template, failedBuilds, totalBuilds int64, templateVersions []map[string]interface{}) {
+			t.Helper()
+
+			require.Equal(t, recipient.ID, notif.UserID)
+			require.Equal(t, notifications.TemplateWorkspaceBuildsFailedReport, notif.TemplateID)
+			require.Equal(t, tmpl.Name, notif.Labels["template_name"])
+			require.Equal(t, tmpl.DisplayName, notif.Labels["template_display_name"])
+			require.Equal(t, failedBuilds, notif.Data["failed_builds"])
+			require.Equal(t, totalBuilds, notif.Data["total_builds"])
+			require.Equal(t, "week", notif.Data["report_frequency"])
+			require.Equal(t, templateVersions, notif.Data["template_versions"])
+		}
+
+		// Setup
+		ctx, logger, db, ps, notifEnq, clk := setup(t)
+
+		// Given
+
+		// Organization
+		org := dbgen.Organization(t, db, database.Organization{})
+
+		// Template admins
+		templateAdmin1 := dbgen.User(t, db, database.User{Username: "template-admin-1", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: templateAdmin1.ID, OrganizationID: org.ID})
+		templateAdmin2 := dbgen.User(t, db, database.User{Username: "template-admin-2", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: templateAdmin2.ID, OrganizationID: org.ID})
+		_ = dbgen.User(t, db, database.User{Name: "template-admin-3", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		// template admin in some other org, they should not receive any notification
+
+		// Regular users
+		user1 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user1.ID, OrganizationID: org.ID})
+		user2 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user2.ID, OrganizationID: org.ID})
+
+		// Templates
+		t1 := dbgen.Template(t, db, database.Template{Name: "template-1", DisplayName: "First Template", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID})
+		t2 := dbgen.Template(t, db, database.Template{Name: "template-2", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Template versions
+		t1v1 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-1", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+		t1v2 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-2", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+		t2v1 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-2-version-1", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t2.ID, Valid: true}, JobID: uuid.New()})
+		t2v2 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-2-version-2", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t2.ID, Valid: true}, JobID: uuid.New()})
+
+		// Workspaces
+		w1 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
+		w2 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t2.ID, OwnerID: user2.ID, OrganizationID: org.ID})
+		w3 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
+		w4 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t2.ID, OwnerID: user2.ID, OrganizationID: org.ID})
+
+		// When: first run
+		notifEnq.Clear()
+		err := reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent) // no notifications
+
+		// One week later...
+		clk.Advance(failedWorkspaceBuildsReportFrequency + time.Minute)
+		now := clk.Now()
+
+		// Workspace builds
+		w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v2.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w1wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 3, TemplateVersionID: t1v2.ID, JobID: w1wb3pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		w2wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 4, TemplateVersionID: t2v1.ID, JobID: w2wb1pj.ID, CreatedAt: now.Add(-5 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w2wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-4 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 5, TemplateVersionID: t2v2.ID, JobID: w2wb2pj.ID, CreatedAt: now.Add(-4 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w2wb3pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w2.ID, BuildNumber: 6, TemplateVersionID: t2v2.ID, JobID: w2wb3pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		w3wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-3 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w3.ID, BuildNumber: 7, TemplateVersionID: t1v1.ID, JobID: w3wb1pj.ID, CreatedAt: now.Add(-3 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		w4wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 8, TemplateVersionID: t2v1.ID, JobID: w4wb1pj.ID, CreatedAt: now.Add(-6 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w4wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w4.ID, BuildNumber: 9, TemplateVersionID: t2v2.ID, JobID: w4wb2pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, authedDB(t, db, logger), notifEnq, clk)
+
+		// Then
+		require.NoError(t, err)
+
+		require.Len(t, notifEnq.Sent, 4) // 2 templates, 2 template admins
+		for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
+			verifyNotification(t, templateAdmin, notifEnq.Sent[i], t1, 3, 4, []map[string]interface{}{
+				{
+					"failed_builds": []map[string]interface{}{
+						{"build_number": int32(7), "workspace_name": w3.Name, "workspace_owner_username": user1.Username},
+						{"build_number": int32(1), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					},
+					"failed_count":          2,
+					"template_version_name": t1v1.Name,
+				},
+				{
+					"failed_builds": []map[string]interface{}{
+						{"build_number": int32(3), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					},
+					"failed_count":          1,
+					"template_version_name": t1v2.Name,
+				},
+			})
+		}
+
+		for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
+			verifyNotification(t, templateAdmin, notifEnq.Sent[i+2], t2, 3, 5, []map[string]interface{}{
+				{
+					"failed_builds": []map[string]interface{}{
+						{"build_number": int32(8), "workspace_name": w4.Name, "workspace_owner_username": user2.Username},
+					},
+					"failed_count":          1,
+					"template_version_name": t2v1.Name,
+				},
+				{
+					"failed_builds": []map[string]interface{}{
+						{"build_number": int32(6), "workspace_name": w2.Name, "workspace_owner_username": user2.Username},
+						{"build_number": int32(5), "workspace_name": w2.Name, "workspace_owner_username": user2.Username},
+					},
+					"failed_count":          2,
+					"template_version_name": t2v2.Name,
+				},
+			})
+		}
+
+		// Given: 6 days later (less than report frequency), and failed build
+		clk.Advance(6 * dayDuration).MustWait(context.Background())
+		now = clk.Now()
+
+		w1wb4pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: now.Add(-dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 77, TemplateVersionID: t1v2.ID, JobID: w1wb4pj.ID, CreatedAt: now.Add(-dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, authedDB(t, db, logger), notifEnq, clk)
+		require.NoError(t, err)
+
+		// Then: no notifications as it is too early
+		require.Empty(t, notifEnq.Sent)
+
+		// Given: 1 day 1 hour later
+		clk.Advance(dayDuration + time.Hour).MustWait(context.Background())
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, authedDB(t, db, logger), notifEnq, clk)
+		require.NoError(t, err)
+
+		// Then: we should see the failed job in the report
+		require.Len(t, notifEnq.Sent, 2) // a new failed job should be reported
+		for i, templateAdmin := range []database.User{templateAdmin1, templateAdmin2} {
+			verifyNotification(t, templateAdmin, notifEnq.Sent[i], t1, 1, 1, []map[string]interface{}{
+				{
+					"failed_builds": []map[string]interface{}{
+						{"build_number": int32(77), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					},
+					"failed_count":          1,
+					"template_version_name": t1v2.Name,
+				},
+			})
+		}
+	})
+
+	t.Run("TooManyFailedBuilds_SecondRun_Report", func(t *testing.T) {
+		t.Parallel()
+
+		verifyNotification := func(t *testing.T, recipient database.User, notif *testutil.Notification, tmpl database.Template, failedBuilds, totalBuilds int64, templateVersions []map[string]interface{}) {
+			t.Helper()
+
+			require.Equal(t, recipient.ID, notif.UserID)
+			require.Equal(t, notifications.TemplateWorkspaceBuildsFailedReport, notif.TemplateID)
+			require.Equal(t, tmpl.Name, notif.Labels["template_name"])
+			require.Equal(t, tmpl.DisplayName, notif.Labels["template_display_name"])
+			require.Equal(t, failedBuilds, notif.Data["failed_builds"])
+			require.Equal(t, totalBuilds, notif.Data["total_builds"])
+			require.Equal(t, "week", notif.Data["report_frequency"])
+			require.Equal(t, templateVersions, notif.Data["template_versions"])
+		}
+
+		// Setup
+		ctx, logger, db, ps, notifEnq, clk := setup(t)
+
+		// Given
+
+		// Organization
+		org := dbgen.Organization(t, db, database.Organization{})
+
+		// Template admins
+		templateAdmin1 := dbgen.User(t, db, database.User{Username: "template-admin-1", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Regular users
+		user1 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user1.ID, OrganizationID: org.ID})
+
+		// Templates
+		t1 := dbgen.Template(t, db, database.Template{Name: "template-1", DisplayName: "First Template", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Template versions
+		t1v1 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-1", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+		t1v2 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-2", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+
+		// Workspaces
+		w1 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
+
+		// When: first run
+		notifEnq.Clear()
+		err := reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent) // no notifications
+
+		// One week later...
+		clk.Advance(failedWorkspaceBuildsReportFrequency + time.Minute)
+		now := clk.Now()
+
+		// Workspace builds
+		pj0 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-24 * time.Hour), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 777, TemplateVersionID: t1v1.ID, JobID: pj0.ID, CreatedAt: now.Add(-24 * time.Hour), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		for i := 1; i <= 23; i++ {
+			at := now.Add(-time.Duration(i) * time.Hour)
+
+			pj1 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}})
+			_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i), TemplateVersionID: t1v1.ID, JobID: pj1.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+			pj2 := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, Error: jobError, ErrorCode: jobErrorCode, CompletedAt: sql.NullTime{Time: at, Valid: true}})
+			_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: int32(i) + 100, TemplateVersionID: t1v2.ID, JobID: pj2.ID, CreatedAt: at, Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		}
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, authedDB(t, db, logger), notifEnq, clk)
+
+		// Then
+		require.NoError(t, err)
+
+		require.Len(t, notifEnq.Sent, 1) // 1 template, 1 template admin
+		verifyNotification(t, templateAdmin1, notifEnq.Sent[0], t1, 46, 47, []map[string]interface{}{
+			{
+				"failed_builds": []map[string]interface{}{
+					{"build_number": int32(23), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(22), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(21), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(20), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(19), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(18), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(17), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(16), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(15), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(14), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+				},
+				"failed_count":          23,
+				"template_version_name": t1v1.Name,
+			},
+			{
+				"failed_builds": []map[string]interface{}{
+					{"build_number": int32(123), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(122), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(121), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(120), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(119), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(118), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(117), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(116), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(115), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+					{"build_number": int32(114), "workspace_name": w1.Name, "workspace_owner_username": user1.Username},
+				},
+				"failed_count":          23,
+				"template_version_name": t1v2.Name,
+			},
+		})
+	})
+
+	t.Run("NoFailedBuilds_NoReport", func(t *testing.T) {
+		t.Parallel()
+
+		// Setup
+		ctx, logger, db, ps, notifEnq, clk := setup(t)
+
+		// Given
+		// Organization
+		org := dbgen.Organization(t, db, database.Organization{})
+
+		// Template admins
+		templateAdmin1 := dbgen.User(t, db, database.User{Username: "template-admin-1", RBACRoles: []string{rbac.RoleTemplateAdmin().Name}})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Regular users
+		user1 := dbgen.User(t, db, database.User{})
+		_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user1.ID, OrganizationID: org.ID})
+
+		// Templates
+		t1 := dbgen.Template(t, db, database.Template{Name: "template-1", DisplayName: "First Template", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID})
+
+		// Template versions
+		t1v1 := dbgen.TemplateVersion(t, db, database.TemplateVersion{Name: "template-1-version-1", CreatedBy: templateAdmin1.ID, OrganizationID: org.ID, TemplateID: uuid.NullUUID{UUID: t1.ID, Valid: true}, JobID: uuid.New()})
+
+		// Workspaces
+		w1 := dbgen.Workspace(t, db, database.Workspace{TemplateID: t1.ID, OwnerID: user1.ID, OrganizationID: org.ID})
+
+		// When: first run
+		notifEnq.Clear()
+		err := reportFailedWorkspaceBuilds(ctx, logger, db, notifEnq, clk)
+
+		// Then: no notifications
+		require.NoError(t, err)
+		require.Empty(t, notifEnq.Sent)
+
+		// Given: one week later, and a successful few jobs being executed
+		clk.Advance(failedWorkspaceBuildsReportFrequency + time.Minute)
+		now := clk.Now()
+
+		// Workspace builds
+		w1wb1pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-6 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 1, TemplateVersionID: t1v1.ID, JobID: w1wb1pj.ID, CreatedAt: now.Add(-2 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+		w1wb2pj := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{OrganizationID: org.ID, CompletedAt: sql.NullTime{Time: now.Add(-5 * dayDuration), Valid: true}})
+		_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{WorkspaceID: w1.ID, BuildNumber: 2, TemplateVersionID: t1v1.ID, JobID: w1wb2pj.ID, CreatedAt: now.Add(-1 * dayDuration), Transition: database.WorkspaceTransitionStart, Reason: database.BuildReasonInitiator})
+
+		// When
+		notifEnq.Clear()
+		err = reportFailedWorkspaceBuilds(ctx, logger, authedDB(t, db, logger), notifEnq, clk)
+
+		// Then: no failures? nothing to report
+		require.NoError(t, err)
+		require.Len(t, notifEnq.Sent, 0) // all jobs succeeded so nothing to report
+	})
+}
+
+func setup(t *testing.T) (context.Context, slog.Logger, database.Store, pubsub.Pubsub, *testutil.FakeNotificationsEnqueuer, *quartz.Mock) {
+	t.Helper()
+
+	// nolint:gocritic // reportFailedWorkspaceBuilds is called by system.
+	ctx := dbauthz.AsSystemRestricted(context.Background())
+	logger := slogtest.Make(t, &slogtest.Options{})
+	db, ps := dbtestutil.NewDB(t)
+	notifyEnq := &testutil.FakeNotificationsEnqueuer{}
+	clk := quartz.NewMock(t)
+	return ctx, logger, db, ps, notifyEnq, clk
+}
+
+func authedDB(t *testing.T, db database.Store, logger slog.Logger) database.Store {
+	t.Helper()
+	return dbauthz.New(db, rbac.NewAuthorizer(prometheus.NewRegistry()), logger, coderdtest.AccessControlStorePointer())
+}
@@ -33,4 +33,5 @@ type Handler interface {
 // Enqueuer enqueues a new notification message in the store and returns its ID, should it enqueue without failure.
 type Enqueuer interface {
 	Enqueue(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error)
+	EnqueueWithData(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, data map[string]any, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error)
 }
@@ -0,0 +1,17 @@
+Hi Bobby,
+
+Template **Bobby First Template** has failed to build 4/55 times over the last week.
+
+**Report:**
+
+**bobby-template-version-1** failed 3 times:
+
+* [mtojek / workspace-1 / #1234](http://test.com/@mtojek/workspace-1/builds/1234)
+* [johndoe / my-workspace-3 / #5678](http://test.com/@johndoe/my-workspace-3/builds/5678)
+* [jack / workwork / #774](http://test.com/@jack/workwork/builds/774)
+
+**bobby-template-version-2** failed 1 time:
+
+* [ben / cool-workspace / #8888](http://test.com/@ben/cool-workspace/builds/8888)
+
+We recommend reviewing these issues to ensure future builds are successful.
@@ -0,0 +1 @@
+Workspace builds failed for template "Bobby First Template"
@@ -17,4 +17,5 @@ type MessagePayload struct {

 	Actions []TemplateAction  `json:"actions"`
 	Labels  map[string]string `json:"labels"`
+	Data    map[string]any    `json:"data"`
 }
@@ -15,11 +15,16 @@ type FakeNotificationsEnqueuer struct {
 type Notification struct {
 	UserID, TemplateID uuid.UUID
 	Labels             map[string]string
+	Data               map[string]any
 	CreatedBy          string
 	Targets            []uuid.UUID
 }

-func (f *FakeNotificationsEnqueuer) Enqueue(_ context.Context, userID, templateID uuid.UUID, labels map[string]string, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
+func (f *FakeNotificationsEnqueuer) Enqueue(ctx context.Context, userID, templateID uuid.UUID, labels map[string]string, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
+	return f.EnqueueWithData(ctx, userID, templateID, labels, nil, createdBy, targets...)
+}
+
+func (f *FakeNotificationsEnqueuer) EnqueueWithData(_ context.Context, userID, templateID uuid.UUID, labels map[string]string, data map[string]any, createdBy string, targets ...uuid.UUID) (*uuid.UUID, error) {
 	f.mu.Lock()
 	defer f.mu.Unlock()

@@ -27,6 +32,7 @@ func (f *FakeNotificationsEnqueuer) Enqueue(_ context.Context, userID, templateI
 		UserID:     userID,
 		TemplateID: templateID,
 		Labels:     labels,
+		Data:       data,
 		CreatedBy:  createdBy,
 		Targets:    targets,
 	})
				`@@ -0,0 +1 @@`
				`Workspace builds failed for template "Bobby First Template"`