fix: fix flake in TestExecutorAutostartSkipsWhenNoProvisionersAvailable (#19478)

The flake here had two causes:
1. related to usage of time.Now() in MustWaitForProvisionersAvailable
and
2. the fact that UpdateProvisionerLastSeenAt can not use a time that is
further in the past than the current LastSeenAt time

Previously the test here was calling
`coderdtest.MustWaitForProvisionersAvailable` which was using `time.Now`
rather than the next tick time like the real `hasProvisionersAvailable`
function does. Additionally, when using `UpdateProvisionerLastSeenAt`
the underlying db query enforces that the time we're trying to set
`LastSeenAt` to cannot be older than the current value.

I was able to reliably reproduce the flake by executing both the
`UpdateProvisionerLastSeenAt` call and `tickCh <- next` in their own
goroutines, the former with a small sleep to reliably ensure we'd
trigger the autobuild before we set the `LastSeenAt` time. That's when I
also noticed that `coderdtest.MustWaitForProvisionersAvailable` was
using `time.Now` instead of the tick time. When I updated that function
to take in a tick time + added a 2nd call to
`UpdateProvisionerLastSeenAt` to set an original non-stale time, we
could then never get the test to pass because the later call to set the
stale time would not actually modify `LastSeenAt`. On top of that,
calling the provisioner daemons closer in the middle of the function
doesn't really do anything of value in this test.

**The fix for the flake is to keep the go routines, ensuring there would
be a flake if there was not a relevant fix, but to include the fix which
is to ensure that we explicitly wait for the provisioner to be stale
before passing the time to `tickCh`.**

---------

Signed-off-by: Callum Styan <callumstyan@gmail.com>

coderd/autobuild/lifecycle_executor_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"errors"
+	"sync"
 	"testing"
 	"time"
 
@@ -1720,19 +1721,32 @@ func TestExecutorAutostartSkipsWhenNoProvisionersAvailable(t *testing.T) {
 	// Stop the workspace while provisioner is available
 	workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop)
 
-	// Wait for provisioner to be available for this specific workspace
-	coderdtest.MustWaitForProvisionersAvailable(t, db, workspace)
 	p, err := coderdtest.GetProvisionerForTags(db, time.Now(), workspace.OrganizationID, provisionerDaemonTags)
 	require.NoError(t, err, "Error getting provisioner for workspace")
 
-	daemon1Closer.Close()
+	var wg sync.WaitGroup
+	wg.Add(2)
 
-	// Ensure the provisioner is stale
-	staleTime := sched.Next(workspace.LatestBuild.CreatedAt).Add((-1 * provisionerdserver.StaleInterval) + -10*time.Second)
-	coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, staleTime)
+	next := sched.Next(workspace.LatestBuild.CreatedAt)
+	go func() {
+		defer wg.Done()
+		// Ensure the provisioner is stale
+		staleTime := next.Add(-(provisionerdserver.StaleInterval * 2))
+		coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, staleTime)
+		p, err = coderdtest.GetProvisionerForTags(db, time.Now(), workspace.OrganizationID, provisionerDaemonTags)
+		assert.NoError(t, err, "Error getting provisioner for workspace")
+		assert.Eventually(t, func() bool { return p.LastSeenAt.Time.UnixNano() == staleTime.UnixNano() }, testutil.WaitMedium, testutil.IntervalFast)
+	}()
 
-	// Trigger autobuild
-	tickCh <- sched.Next(workspace.LatestBuild.CreatedAt)
+	go func() {
+		defer wg.Done()
+		// Ensure the provisioner is gone or stale before triggering the autobuild
+		coderdtest.MustWaitForProvisionersUnavailable(t, db, workspace, provisionerDaemonTags, next)
+		// Trigger autobuild
+		tickCh <- next
+	}()
+
+	wg.Wait()
 
 	stats := <-statsCh
 
@@ -1758,5 +1772,5 @@ func TestExecutorAutostartSkipsWhenNoProvisionersAvailable(t *testing.T) {
 	}()
 	stats = <-statsCh
 
-	assert.Len(t, stats.Transitions, 1, "should not create builds when no provisioners available")
+	assert.Len(t, stats.Transitions, 1, "should create builds when provisioners are available")
 }