diff --git a/cli/agent.go b/cli/agent.go
index c0bccc7769..5d7738d3f4 100644
--- a/cli/agent.go
+++ b/cli/agent.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -201,18 +202,15 @@ func workspaceAgent() *serpent.Command {
 			// Enable pprof handler
 			// This prevents the pprof import from being accidentally deleted.
 			_ = pprof.Handler
-			pprofSrvClose := ServeHandler(ctx, logger, nil, pprofAddress, "pprof")
-			defer pprofSrvClose()
-			if port, err := extractPort(pprofAddress); err == nil {
-				ignorePorts[port] = "pprof"
-			}
+			if pprofAddress != "" {
+				pprofSrvClose := ServeHandler(ctx, logger, nil, pprofAddress, "pprof")
+				defer pprofSrvClose()
 
-			if port, err := extractPort(prometheusAddress); err == nil {
-				ignorePorts[port] = "prometheus"
-			}
-
-			if port, err := extractPort(debugAddress); err == nil {
-				ignorePorts[port] = "debug"
+				if port, err := extractPort(pprofAddress); err == nil {
+					ignorePorts[port] = "pprof"
+				}
+			} else {
+				logger.Debug(ctx, "pprof address is empty, disabling pprof server")
 			}
 
 			executablePath, err := os.Executable()
@@ -276,6 +274,28 @@ func workspaceAgent() *serpent.Command {
 			for {
 				prometheusRegistry := prometheus.NewRegistry()
 
+				promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
+				var serverClose []func()
+				if prometheusAddress != "" {
+					prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
+					serverClose = append(serverClose, prometheusSrvClose)
+
+					if port, err := extractPort(prometheusAddress); err == nil {
+						ignorePorts[port] = "prometheus"
+					}
+				} else {
+					logger.Debug(ctx, "prometheus address is empty, disabling prometheus server")
+				}
+
+				if debugAddress != "" {
+					// ServerHandle depends on `agnt.HTTPDebug()`, but `agnt`
+					// depends on `ignorePorts`. Keep this if statement in sync
+					// with below.
+					if port, err := extractPort(debugAddress); err == nil {
+						ignorePorts[port] = "debug"
+					}
+				}
+
 				agnt := agent.New(agent.Options{
 					Client:        client,
 					Logger:        logger,
@@ -299,10 +319,15 @@ func workspaceAgent() *serpent.Command {
 					},
 				})
 
-				promHandler := agent.PrometheusMetricsHandler(prometheusRegistry, logger)
-				prometheusSrvClose := ServeHandler(ctx, logger, promHandler, prometheusAddress, "prometheus")
-
-				debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
+				if debugAddress != "" {
+					// ServerHandle depends on `agnt.HTTPDebug()`, but `agnt`
+					// depends on `ignorePorts`. Keep this if statement in sync
+					// with above.
+					debugSrvClose := ServeHandler(ctx, logger, agnt.HTTPDebug(), debugAddress, "debug")
+					serverClose = append(serverClose, debugSrvClose)
+				} else {
+					logger.Debug(ctx, "debug address is empty, disabling debug server")
+				}
 
 				select {
 				case <-ctx.Done():
@@ -314,8 +339,11 @@ func workspaceAgent() *serpent.Command {
 				}
 
 				lastErr = agnt.Close()
-				debugSrvClose()
-				prometheusSrvClose()
+
+				slices.Reverse(serverClose)
+				for _, closeFunc := range serverClose {
+					closeFunc()
+				}
 
 				if mustExit {
 					break
diff --git a/cli/agent_test.go b/cli/agent_test.go
index b0b8cbcc97..0d0594d8a6 100644
--- a/cli/agent_test.go
+++ b/cli/agent_test.go
@@ -178,6 +178,51 @@ func TestWorkspaceAgent(t *testing.T) {
 		require.Greater(t, atomic.LoadInt64(&called), int64(0), "expected coderd to be reached with custom headers")
 		require.Greater(t, atomic.LoadInt64(&derpCalled), int64(0), "expected /derp to be called with custom headers")
 	})
+
+	t.Run("DisabledServers", func(t *testing.T) {
+		t.Parallel()
+
+		client, db := coderdtest.NewWithDatabase(t, nil)
+		user := coderdtest.CreateFirstUser(t, client)
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).WithAgent().Do()
+
+		logDir := t.TempDir()
+		inv, _ := clitest.New(t,
+			"agent",
+			"--auth", "token",
+			"--agent-token", r.AgentToken,
+			"--agent-url", client.URL.String(),
+			"--log-dir", logDir,
+			"--pprof-address", "",
+			"--prometheus-address", "",
+			"--debug-address", "",
+		)
+
+		clitest.Start(t, inv)
+
+		// Verify the agent is connected and working.
+		resources := coderdtest.NewWorkspaceAgentWaiter(t, client, r.Workspace.ID).
+			MatchResources(matchAgentWithVersion).Wait()
+		require.Len(t, resources, 1)
+		require.Len(t, resources[0].Agents, 1)
+		require.NotEmpty(t, resources[0].Agents[0].Version)
+
+		// Verify the servers are not listening by checking the log for disabled
+		// messages.
+		require.Eventually(t, func() bool {
+			logContent, err := os.ReadFile(filepath.Join(logDir, "coder-agent.log"))
+			if err != nil {
+				return false
+			}
+			logStr := string(logContent)
+			return strings.Contains(logStr, "pprof address is empty, disabling pprof server") &&
+				strings.Contains(logStr, "prometheus address is empty, disabling prometheus server") &&
+				strings.Contains(logStr, "debug address is empty, disabling debug server")
+		}, testutil.WaitLong, testutil.IntervalMedium)
+	})
 }
 
 func matchAgentWithVersion(rs []codersdk.WorkspaceResource) bool {