fix: do not warn on valid known experiments (#18514)

Fixes https://github.com/coder/coder/issues/18024

* drive-by: renames `handleExperimentsSafe` to
`handleExperimentsAvailable` to better match semantics
* defines list of `codersdk.ExperimentsKnown` and updates
`ReadExperiments` to log on invalid experiments
* typescript-ignores `codersdk.Experiments` so apitypings generates a
valid enum list of possible values of experiment
* updates OverviewPageView to distinguish between known 'hidden'
experiments and unknown 'invalid' experiments

coderd/coderd.go

@@ -972,7 +972,7 @@ func New(options *Options) *API {
 		})
 		r.Route("/experiments", func(r chi.Router) {
 			r.Use(apiKeyMiddleware)
-			r.Get("/available", handleExperimentsSafe)
+			r.Get("/available", handleExperimentsAvailable)
 			r.Get("/", api.handleExperimentsGet)
 		})
 		r.Get("/updatecheck", api.updateCheck)
@@ -1895,7 +1895,9 @@ func ReadExperiments(log slog.Logger, raw []string) codersdk.Experiments {
 			exps = append(exps, codersdk.ExperimentsSafe...)
 		default:
 			ex := codersdk.Experiment(strings.ToLower(v))
-			if !slice.Contains(codersdk.ExperimentsSafe, ex) {
+			if !slice.Contains(codersdk.ExperimentsKnown, ex) {
+				log.Warn(context.Background(), "ignoring unknown experiment", slog.F("experiment", ex))
+			} else if !slice.Contains(codersdk.ExperimentsSafe, ex) {
 				log.Warn(context.Background(), "🐉 HERE BE DRAGONS: opting into hidden experiment", slog.F("experiment", ex))
 			}
 			exps = append(exps, ex)

coderd/experiments.go

@@ -26,7 +26,7 @@ func (api *API) handleExperimentsGet(rw http.ResponseWriter, r *http.Request) {
 // @Tags General
 // @Success 200 {array} codersdk.Experiment
 // @Router /experiments/available [get]
-func handleExperimentsSafe(rw http.ResponseWriter, r *http.Request) {
+func handleExperimentsAvailable(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	httpapi.Write(ctx, rw, http.StatusOK, codersdk.AvailableExperiments{
 		Safe: codersdk.ExperimentsSafe,

codersdk/deployment.go

@@ -3372,6 +3372,18 @@ const (
 	ExperimentAITasks            Experiment = "ai-tasks"             // Enables the new AI tasks feature.
 )
 
+// ExperimentsKnown should include all experiments defined above.
+var ExperimentsKnown = Experiments{
+	ExperimentExample,
+	ExperimentAutoFillParameters,
+	ExperimentNotifications,
+	ExperimentWorkspaceUsage,
+	ExperimentWebPush,
+	ExperimentWorkspacePrebuilds,
+	ExperimentAgenticChat,
+	ExperimentAITasks,
+}
+
 // ExperimentsSafe should include all experiments that are safe for
 // users to opt-in to via --experimental='*'.
 // Experiments that are not ready for consumption by all users should
@@ -3384,6 +3396,9 @@ var ExperimentsSafe = Experiments{
 // Multiple experiments may be enabled at the same time.
 // Experiments are not safe for production use, and are not guaranteed to
 // be backwards compatible. They may be removed or renamed at any time.
+// The below typescript-ignore annotation allows our typescript generator
+// to generate an enum list, which is used in the frontend.
+// @typescript-ignore Experiments
 type Experiments []Experiment
 
 // Returns a list of experiments that are enabled for the deployment.

site/src/api/queries/experiments.ts

@@ -1,11 +1,11 @@
 import { API } from "api/api";
-import type { Experiments } from "api/typesGenerated";
+import { type Experiment, Experiments } from "api/typesGenerated";
 import type { MetadataState } from "hooks/useEmbeddedMetadata";
 import { cachedQuery } from "./util";
 
 const experimentsKey = ["experiments"] as const;
 
-export const experiments = (metadata: MetadataState<Experiments>) => {
+export const experiments = (metadata: MetadataState<Experiment[]>) => {
 	return cachedQuery({
 		metadata,
 		queryKey: experimentsKey,
@@ -19,3 +19,7 @@ export const availableExperiments = () => {
 		queryFn: async () => API.getAvailableExperiments(),
 	};
 };
+
+export const isKnownExperiment = (experiment: string): boolean => {
+	return Experiments.includes(experiment as Experiment);
+};

site/src/api/typesGenerated.ts

@@ -835,8 +835,16 @@ export type Experiment =
 	| "workspace-prebuilds"
 	| "workspace-usage";
 
-// From codersdk/deployment.go
-export type Experiments = readonly Experiment[];
+export const Experiments: Experiment[] = [
+	"ai-tasks",
+	"agentic-chat",
+	"auto-fill-parameters",
+	"example",
+	"notifications",
+	"web-push",
+	"workspace-prebuilds",
+	"workspace-usage",
+];
 
 // From codersdk/externalauth.go
 export interface ExternalAuth {

site/src/hooks/useEmbeddedMetadata.ts

@@ -2,7 +2,7 @@ import type {
 	AppearanceConfig,
 	BuildInfoResponse,
 	Entitlements,
-	Experiments,
+	Experiment,
 	Region,
 	User,
 	UserAppearanceSettings,
@@ -24,7 +24,7 @@ export const DEFAULT_METADATA_KEY = "property";
  */
 type AvailableMetadata = Readonly<{
 	user: User;
-	experiments: Experiments;
+	experiments: Experiment[];
 	appearance: AppearanceConfig;
 	userAppearance: UserAppearanceSettings;
 	entitlements: Entitlements;
@@ -89,7 +89,7 @@ export class MetadataManager implements MetadataManagerApi {
 			userAppearance:
 				this.registerValue<UserAppearanceSettings>("userAppearance"),
 			entitlements: this.registerValue<Entitlements>("entitlements"),
-			experiments: this.registerValue<Experiments>("experiments"),
+			experiments: this.registerValue<Experiment[]>("experiments"),
 			"build-info": this.registerValue<BuildInfoResponse>("build-info"),
 			regions: this.registerRegionValue(),
 			tasksTabVisible: this.registerValue<boolean>("tasksTabVisible"),

site/src/modules/dashboard/DashboardProvider.tsx

@@ -5,7 +5,7 @@ import { organizations } from "api/queries/organizations";
 import type {
 	AppearanceConfig,
 	Entitlements,
-	Experiments,
+	Experiment,
 	Organization,
 } from "api/typesGenerated";
 import { ErrorAlert } from "components/Alert/ErrorAlert";
@@ -19,7 +19,7 @@ import { selectFeatureVisibility } from "./entitlements";
 
 export interface DashboardValue {
 	entitlements: Entitlements;
-	experiments: Experiments;
+	experiments: Experiment[];
 	appearance: AppearanceConfig;
 	organizations: readonly Organization[];
 	showOrganizations: boolean;

site/src/pages/DeploymentSettingsPage/OverviewPage/OverviewPage.tsx

@@ -1,5 +1,9 @@
 import { deploymentDAUs } from "api/queries/deployment";
-import { availableExperiments, experiments } from "api/queries/experiments";
+import {
+	availableExperiments,
+	experiments,
+	isKnownExperiment,
+} from "api/queries/experiments";
 import { useEmbeddedMetadata } from "hooks/useEmbeddedMetadata";
 import { useDeploymentConfig } from "modules/management/DeploymentConfigProvider";
 import type { FC } from "react";
@@ -18,7 +22,7 @@ const OverviewPage: FC = () => {
 	const safeExperiments = safeExperimentsQuery.data?.safe ?? [];
 	const invalidExperiments =
 		enabledExperimentsQuery.data?.filter((exp) => {
-			return !safeExperiments.includes(exp);
+			return !isKnownExperiment(exp);
 		}) ?? [];
 
 	const { data: dailyActiveUsers } = useQuery(deploymentDAUs());

site/src/pages/DeploymentSettingsPage/OverviewPage/OverviewPageView.stories.tsx

@@ -30,7 +30,7 @@ const meta: Meta<typeof OverviewPageView> = {
 				description:
 					"Enable one or more experiments. These are not ready for production. Separate multiple experiments with commas, or enter '*' to opt-in to all available experiments.",
 				flag: "experiments",
-				value: ["workspace_actions"],
+				value: ["example"],
 				flag_shorthand: "",
 				hidden: false,
 			},
@@ -82,8 +82,8 @@ export const allExperimentsEnabled: Story = {
 				hidden: false,
 			},
 		],
-		safeExperiments: ["shared-ports"],
-		invalidExperiments: ["invalid"],
+		safeExperiments: ["example"],
+		invalidExperiments: [],
 	},
 };
 
@@ -118,7 +118,7 @@ export const invalidExperimentsEnabled: Story = {
 				hidden: false,
 			},
 		],
-		safeExperiments: ["shared-ports"],
+		safeExperiments: ["example"],
 		invalidExperiments: ["invalid"],
 	},
 };

site/src/pages/DeploymentSettingsPage/OverviewPage/OverviewPageView.tsx

@@ -1,7 +1,7 @@
 import AlertTitle from "@mui/material/AlertTitle";
 import type {
 	DAUsResponse,
-	Experiments,
+	Experiment,
 	SerpentOption,
 } from "api/typesGenerated";
 import { Link } from "components/Link/Link";
@@ -22,8 +22,8 @@ import { UserEngagementChart } from "./UserEngagementChart";
 type OverviewPageViewProps = {
 	deploymentOptions: SerpentOption[];
 	dailyActiveUsers: DAUsResponse | undefined;
-	readonly invalidExperiments: Experiments | string[];
-	readonly safeExperiments: Experiments | string[];
+	readonly invalidExperiments: readonly string[];
+	readonly safeExperiments: readonly Experiment[];
 };
 
 export const OverviewPageView: FC<OverviewPageViewProps> = ({