chore: rename service -> coder_service, remove agent_id label (#19241)

Pyroscope uses `service` tag for top level distinction. So move our
`service` -> `coder_service`

coderd/coderd.go

@@ -14,12 +14,14 @@ import (
 	"net/url"
 	"path/filepath"
 	"regexp"
+	"runtime/pprof"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/coder/coder/v2/coderd/oauth2provider"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	"github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
 
@@ -1340,7 +1342,13 @@ func New(options *Options) *API {
 			).Get("/connection", api.workspaceAgentConnectionGeneric)
 			r.Route("/me", func(r chi.Router) {
 				r.Use(workspaceAgentInfo)
-				r.Get("/rpc", api.workspaceAgentRPC)
+				r.Group(func(r chi.Router) {
+					r.Use(
+						// Override the request_type for agent rpc traffic.
+						httpmw.WithStaticProfilingLabels(pprof.Labels(pproflabel.RequestTypeTag, "agent-rpc")),
+					)
+					r.Get("/rpc", api.workspaceAgentRPC)
+				})
 				r.Patch("/logs", api.patchWorkspaceAgentLogs)
 				r.Patch("/app-status", api.patchWorkspaceAgentAppStatus)
 				// Deprecated: Required to support legacy agents

coderd/httpmw/pprof.go

@@ -22,9 +22,22 @@ func WithProfilingLabels(next http.Handler) http.Handler {
 			requestType = "websocket"
 		}
 
-		pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, "request_type", requestType), func(ctx context.Context) {
+		pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, pproflabel.RequestTypeTag, requestType), func(ctx context.Context) {
 			r = r.WithContext(ctx)
 			next.ServeHTTP(rw, r)
 		})
 	})
 }
+
+func WithStaticProfilingLabels(labels pprof.LabelSet) func(next http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			ctx := r.Context()
+
+			pprof.Do(ctx, labels, func(ctx context.Context) {
+				r = r.WithContext(ctx)
+				next.ServeHTTP(rw, r)
+			})
+		})
+	}
+}

coderd/pproflabel/pproflabel.go

@@ -10,14 +10,22 @@ func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) {
 	go pprof.Do(ctx, labels, f)
 }
 
+func Do(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) {
+	pprof.Do(ctx, labels, f)
+}
+
 const (
-	ServiceTag = "service"
+	// ServiceTag should not collide with the pyroscope built-in tag "service".
+	// Use `coder_` to avoid collisions.
+	ServiceTag = "coder_service"
 
 	ServiceHTTPServer           = "http-api"
 	ServiceLifecycles           = "lifecycle-executor"
 	ServiceMetricCollector      = "metrics-collector"
 	ServicePrebuildReconciler   = "prebuilds-reconciler"
 	ServiceTerraformProvisioner = "terraform-provisioner"
+
+	RequestTypeTag = "coder_request_type"
 )
 
 func Service(name string, pairs ...string) pprof.LabelSet {

coderd/workspaceagentsrpc.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"runtime/pprof"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -348,16 +347,14 @@ func (m *agentConnectionMonitor) init() {
 func (m *agentConnectionMonitor) start(ctx context.Context) {
 	ctx, m.cancel = context.WithCancel(ctx)
 	m.wg.Add(2)
-	go pprof.Do(ctx, pprof.Labels("agent", m.workspaceAgent.ID.String()),
-		func(ctx context.Context) {
-			defer m.wg.Done()
-			m.sendPings(ctx)
-		})
-	go pprof.Do(ctx, pprof.Labels("agent", m.workspaceAgent.ID.String()),
-		func(ctx context.Context) {
-			defer m.wg.Done()
-			m.monitor(ctx)
-		})
+	go func(ctx context.Context) {
+		defer m.wg.Done()
+		m.sendPings(ctx)
+	}(ctx)
+	go func(ctx context.Context) {
+		defer m.wg.Done()
+		m.monitor(ctx)
+	}(ctx)
 }
 
 func (m *agentConnectionMonitor) monitor(ctx context.Context) {