feat: add backoff to workspace agent polling (#20157)

## Summary

In this pull request we're adding a simple backoff to the workspace
agent polling. This backoff is being added to address seemingly random
cases of elevated number of calls that we've seen to the
`api/v2/workspaceagent/{agent_id}` endpoint.

For more information on the investigation, see:
https://github.com/coder/internal/issues/725

### Changes

- Updated the polling to use predefined progressive intervals for
polling instead of continuously polling every 500ms

### Testing

- Added a test for the function used to calculate the progressive
polling interval

Co-authored-by: Spike Curtis <spike@coder.com>

cli/cliui/agent.go

@@ -53,6 +53,9 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 		t := time.NewTimer(0)
 		defer t.Stop()
 
+		startTime := time.Now()
+		baseInterval := opts.FetchInterval
+
 		for {
 			select {
 			case <-ctx.Done():
@@ -68,7 +71,11 @@ func Agent(ctx context.Context, writer io.Writer, agentID uuid.UUID, opts AgentO
 					return
 				}
 				fetchedAgent <- fetchAgent{agent: agent}
-				t.Reset(opts.FetchInterval)
+
+				// Adjust the interval based on how long we've been waiting.
+				elapsed := time.Since(startTime)
+				currentInterval := GetProgressiveInterval(baseInterval, elapsed)
+				t.Reset(currentInterval)
 			}
 		}
 	}()
@@ -293,6 +300,24 @@ func safeDuration(sw *stageWriter, a, b *time.Time) time.Duration {
 	return a.Sub(*b)
 }
 
+// GetProgressiveInterval returns an interval that increases over time.
+// The interval starts at baseInterval and increases to
+// a maximum of baseInterval * 16 over time.
+func GetProgressiveInterval(baseInterval time.Duration, elapsed time.Duration) time.Duration {
+	switch {
+	case elapsed < 60*time.Second:
+		return baseInterval // 500ms for first 60 seconds
+	case elapsed < 2*time.Minute:
+		return baseInterval * 2 // 1s for next 1 minute
+	case elapsed < 5*time.Minute:
+		return baseInterval * 4 // 2s for next 3 minutes
+	case elapsed < 10*time.Minute:
+		return baseInterval * 8 // 4s for next 5 minutes
+	default:
+		return baseInterval * 16 // 8s after 10 minutes
+	}
+}
+
 type closeFunc func() error
 
 func (c closeFunc) Close() error {

cli/cliui/agent_test.go

@@ -866,3 +866,31 @@ func TestConnDiagnostics(t *testing.T) {
 		})
 	}
 }
+
+func TestGetProgressiveInterval(t *testing.T) {
+	t.Parallel()
+
+	baseInterval := 500 * time.Millisecond
+
+	testCases := []struct {
+		name     string
+		elapsed  time.Duration
+		expected time.Duration
+	}{
+		{"first_minute", 30 * time.Second, baseInterval},
+		{"second_minute", 90 * time.Second, baseInterval * 2},
+		{"third_to_fifth_minute", 3 * time.Minute, baseInterval * 4},
+		{"sixth_to_tenth_minute", 7 * time.Minute, baseInterval * 8},
+		{"after_ten_minutes", 15 * time.Minute, baseInterval * 16},
+		{"boundary_first_minute", 59 * time.Second, baseInterval},
+		{"boundary_second_minute", 61 * time.Second, baseInterval * 2},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			result := cliui.GetProgressiveInterval(baseInterval, tc.elapsed)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}