From 435c67ab75e85c99c6915b8baa45b61baa628f97 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Fri, 7 Jul 2023 09:10:14 +0100 Subject: [PATCH] refactor(cli)!: move scaletest to exp/scaletest (#8339) * refactor(cli): mv scaletest exp/scaletest * make gen --- cli/exp.go | 18 ++ cli/{scaletest.go => exp_scaletest.go} | 2 +- ...caletest_test.go => exp_scaletest_test.go} | 4 +- cli/root.go | 2 +- cli/testdata/coder_--help.golden | 1 - docs/cli.md | 1 - docs/cli/scaletest.md | 19 -- docs/cli/scaletest_cleanup.md | 49 ---- docs/cli/scaletest_create-workspaces.md | 276 ------------------ docs/cli/scaletest_workspace-traffic.md | 159 ---------- docs/manifest.json | 20 -- scaletest/lib/coder_workspacetraffic.sh | 2 +- scaletest/scaletest.sh | 2 +- 13 files changed, 24 insertions(+), 531 deletions(-) create mode 100644 cli/exp.go rename cli/{scaletest.go => exp_scaletest.go} (99%) rename cli/{scaletest_test.go => exp_scaletest_test.go} (93%) delete mode 100644 docs/cli/scaletest.md delete mode 100644 docs/cli/scaletest_cleanup.md delete mode 100644 docs/cli/scaletest_create-workspaces.md delete mode 100644 docs/cli/scaletest_workspace-traffic.md diff --git a/cli/exp.go b/cli/exp.go new file mode 100644 index 0000000000..2513a8fda4 --- /dev/null +++ b/cli/exp.go @@ -0,0 +1,18 @@ +package cli + +import "github.com/coder/coder/cli/clibase" + +func (r *RootCmd) expCmd() *clibase.Cmd { + cmd := &clibase.Cmd{ + Use: "exp", + Short: "Internal commands for testing and experimentation. These are prone to breaking changes with no notice.", + Handler: func(i *clibase.Invocation) error { + return i.Command.HelpHandler(i) + }, + Hidden: true, + Children: []*clibase.Cmd{ + r.scaletestCmd(), + }, + } + return cmd +} diff --git a/cli/scaletest.go b/cli/exp_scaletest.go similarity index 99% rename from cli/scaletest.go rename to cli/exp_scaletest.go index e18f281ea7..2fa3b2c37d 100644 --- a/cli/scaletest.go +++ b/cli/exp_scaletest.go @@ -38,7 +38,7 @@ import ( const scaletestTracerName = "coder_scaletest" -func (r *RootCmd) scaletest() *clibase.Cmd { +func (r *RootCmd) scaletestCmd() *clibase.Cmd { cmd := &clibase.Cmd{ Use: "scaletest", Short: "Run a scale test against the Coder API", diff --git a/cli/scaletest_test.go b/cli/exp_scaletest_test.go similarity index 93% rename from cli/scaletest_test.go rename to cli/exp_scaletest_test.go index 729ad9d0d7..3ff54dc9ab 100644 --- a/cli/scaletest_test.go +++ b/cli/exp_scaletest_test.go @@ -29,7 +29,7 @@ func TestScaleTestCreateWorkspaces(t *testing.T) { tDir := t.TempDir() outputFile := filepath.Join(tDir, "output.json") - inv, root := clitest.New(t, "scaletest", "create-workspaces", + inv, root := clitest.New(t, "exp", "scaletest", "create-workspaces", "--count", "2", "--template", "doesnotexist", "--no-cleanup", @@ -63,7 +63,7 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) { client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) - inv, root := clitest.New(t, "scaletest", "workspace-traffic", + inv, root := clitest.New(t, "exp", "scaletest", "workspace-traffic", "--timeout", "1s", "--bytes-per-tick", "1024", "--tick-interval", "100ms", diff --git a/cli/root.go b/cli/root.go index d7e168657a..09cfb2b534 100644 --- a/cli/root.go +++ b/cli/root.go @@ -95,7 +95,6 @@ func (r *RootCmd) Core() []*clibase.Cmd { r.list(), r.ping(), r.rename(), - r.scaletest(), r.schedules(), r.show(), r.speedtest(), @@ -111,6 +110,7 @@ func (r *RootCmd) Core() []*clibase.Cmd { r.netcheck(), r.vscodeSSH(), r.workspaceAgent(), + r.expCmd(), } } diff --git a/cli/testdata/coder_--help.golden b/cli/testdata/coder_--help.golden index a1eb63de4a..280dbce4ab 100644 --- a/cli/testdata/coder_--help.golden +++ b/cli/testdata/coder_--help.golden @@ -26,7 +26,6 @@ Coder v0.0.0-devel — A tool for provisioning self-hosted development environme reset-password Directly connect to the database to reset a user's password restart Restart a workspace - scaletest Run a scale test against the Coder API schedule Schedule automated start and stop times for workspaces server Start a Coder server show Display details of a workspace's resources and agents diff --git a/docs/cli.md b/docs/cli.md index 14c182f616..847017841d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -42,7 +42,6 @@ Coder — A tool for provisioning self-hosted development environments with Terr | [rename](./cli/rename.md) | Rename a workspace | | [reset-password](./cli/reset-password.md) | Directly connect to the database to reset a user's password | | [restart](./cli/restart.md) | Restart a workspace | -| [scaletest](./cli/scaletest.md) | Run a scale test against the Coder API | | [schedule](./cli/schedule.md) | Schedule automated start and stop times for workspaces | | [server](./cli/server.md) | Start a Coder server | | [show](./cli/show.md) | Display details of a workspace's resources and agents | diff --git a/docs/cli/scaletest.md b/docs/cli/scaletest.md deleted file mode 100644 index dd9eb4c646..0000000000 --- a/docs/cli/scaletest.md +++ /dev/null @@ -1,19 +0,0 @@ - - -# scaletest - -Run a scale test against the Coder API - -## Usage - -```console -coder scaletest -``` - -## Subcommands - -| Name | Purpose | -| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [cleanup](./scaletest_cleanup.md) | Cleanup scaletest workspaces, then cleanup scaletest users. | -| [create-workspaces](./scaletest_create-workspaces.md) | Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard. | -| [workspace-traffic](./scaletest_workspace-traffic.md) | Generate traffic to scaletest workspaces through coderd | diff --git a/docs/cli/scaletest_cleanup.md b/docs/cli/scaletest_cleanup.md deleted file mode 100644 index 29c4a526d9..0000000000 --- a/docs/cli/scaletest_cleanup.md +++ /dev/null @@ -1,49 +0,0 @@ - - -# scaletest cleanup - -Cleanup scaletest workspaces, then cleanup scaletest users. - -## Usage - -```console -coder scaletest cleanup [flags] -``` - -## Description - -```console -The strategy flags will apply to each stage of the cleanup process. -``` - -## Options - -### --cleanup-concurrency - -| | | -| ----------- | ------------------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_CLEANUP_CONCURRENCY | -| Default | 1 | - -Number of concurrent cleanup jobs to run. 0 means unlimited. - -### --cleanup-job-timeout - -| | | -| ----------- | ------------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_JOB_TIMEOUT | -| Default | 5m | - -Timeout per job. Jobs may take longer to complete under higher concurrency limits. - -### --cleanup-timeout - -| | | -| ----------- | --------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_TIMEOUT | -| Default | 30m | - -Timeout for the entire cleanup run. 0 means unlimited. diff --git a/docs/cli/scaletest_create-workspaces.md b/docs/cli/scaletest_create-workspaces.md deleted file mode 100644 index ed4b10f898..0000000000 --- a/docs/cli/scaletest_create-workspaces.md +++ /dev/null @@ -1,276 +0,0 @@ - - -# scaletest create-workspaces - -Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard. - -## Usage - -```console -coder scaletest create-workspaces [flags] -``` - -## Description - -```console -It is recommended that all rate limits are disabled on the server before running this scaletest. This test generates many login events which will be rate limited against the (most likely single) IP. -``` - -## Options - -### --cleanup-concurrency - -| | | -| ----------- | ------------------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_CLEANUP_CONCURRENCY | -| Default | 1 | - -Number of concurrent cleanup jobs to run. 0 means unlimited. - -### --cleanup-job-timeout - -| | | -| ----------- | ------------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_JOB_TIMEOUT | -| Default | 5m | - -Timeout per job. Jobs may take longer to complete under higher concurrency limits. - -### --cleanup-timeout - -| | | -| ----------- | --------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_TIMEOUT | -| Default | 30m | - -Timeout for the entire cleanup run. 0 means unlimited. - -### --concurrency - -| | | -| ----------- | ----------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_CONCURRENCY | -| Default | 1 | - -Number of concurrent jobs to run. 0 means unlimited. - -### --connect-hold - -| | | -| ----------- | ------------------------------------------ | -| Type | duration | -| Environment | $CODER_SCALETEST_CONNECT_HOLD | -| Default | 30s | - -How long to hold the WireGuard connection open for. - -### --connect-interval - -| | | -| ----------- | ---------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CONNECT_INTERVAL | -| Default | 1s | - -How long to wait between making requests to the --connect-url once the connection is established. - -### --connect-mode - -| | | -| ----------- | ------------------------------------------ | -------------- | -| Type | enum[derp | direct] | -| Environment | $CODER_SCALETEST_CONNECT_MODE | -| Default | derp | - -Mode to use for connecting to the workspace. - -### --connect-timeout - -| | | -| ----------- | --------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CONNECT_TIMEOUT | -| Default | 5s | - -Timeout for each request to the --connect-url. - -### --connect-url - -| | | -| ----------- | ----------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_CONNECT_URL | - -URL to connect to inside the the workspace over WireGuard. If not specified, no connections will be made over WireGuard. - -### -c, --count - -| | | -| ----------- | ----------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_COUNT | -| Default | 1 | - -Required: Number of workspaces to create. - -### --job-timeout - -| | | -| ----------- | ----------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_JOB_TIMEOUT | -| Default | 5m | - -Timeout per job. Jobs may take longer to complete under higher concurrency limits. - -### --no-cleanup - -| | | -| ----------- | ---------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_NO_CLEANUP | - -Do not clean up resources after the test completes. You can cleanup manually using coder scaletest cleanup. - -### --no-plan - -| | | -| ----------- | ------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_NO_PLAN | - -Skip the dry-run step to plan the workspace creation. This step ensures that the given parameters are valid for the given template. - -### --no-wait-for-agents - -| | | -| ----------- | ------------------------------------------------ | -| Type | bool | -| Environment | $CODER_SCALETEST_NO_WAIT_FOR_AGENTS | - -Do not wait for agents to start before marking the test as succeeded. This can be useful if you are running the test against a template that does not start the agent quickly. - -### --output - -| | | -| ----------- | ------------------------------------- | -| Type | string-array | -| Environment | $CODER_SCALETEST_OUTPUTS | -| Default | text | - -Output format specs in the format "[:]". Not specifying a path will default to stdout. Available formats: text, json. - -### --run-command - -| | | -| ----------- | ----------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_RUN_COMMAND | - -Command to run inside each workspace using reconnecting-pty (i.e. web terminal protocol). If not specified, no command will be run. - -### --run-expect-output - -| | | -| ----------- | ----------------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_RUN_EXPECT_OUTPUT | - -Expect the command to output the given string (on a single line). If the command does not output the given string, it will be marked as failed. - -### --run-expect-timeout - -| | | -| ----------- | ------------------------------------------------ | -| Type | bool | -| Environment | $CODER_SCALETEST_RUN_EXPECT_TIMEOUT | - -Expect the command to timeout. If the command does not finish within the given --run-timeout, it will be marked as succeeded. If the command finishes before the timeout, it will be marked as failed. - -### --run-log-output - -| | | -| ----------- | -------------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_RUN_LOG_OUTPUT | - -Log the output of the command to the test logs. This should be left off unless you expect small amounts of output. Large amounts of output will cause high memory usage. - -### --run-timeout - -| | | -| ----------- | ----------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_RUN_TIMEOUT | -| Default | 5s | - -Timeout for the command to complete. - -### -t, --template - -| | | -| ----------- | -------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_TEMPLATE | - -Required: Name or ID of the template to use for workspaces. - -### --timeout - -| | | -| ----------- | ------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_TIMEOUT | -| Default | 30m | - -Timeout for the entire test run. 0 means unlimited. - -### --trace - -| | | -| ----------- | ----------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE | - -Whether application tracing data is collected. It exports to a backend configured by environment variables. See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md. - -### --trace-coder - -| | | -| ----------- | ----------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE_CODER | - -Whether opentelemetry traces are sent to Coder. We recommend keeping this disabled unless we advise you to enable it. - -### --trace-honeycomb-api-key - -| | | -| ----------- | ----------------------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_TRACE_HONEYCOMB_API_KEY | - -Enables trace exporting to Honeycomb.io using the provided API key. - -### --trace-propagate - -| | | -| ----------- | --------------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE_PROPAGATE | - -Enables trace propagation to the Coder backend, which will be used to correlate server-side spans with client-side spans. Only enable this if the server is configured with the exact same tracing configuration as the client. - -### --use-host-login - -| | | -| ----------- | -------------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_USE_HOST_LOGIN | -| Default | false | - -Use the use logged in on the host machine, instead of creating users. diff --git a/docs/cli/scaletest_workspace-traffic.md b/docs/cli/scaletest_workspace-traffic.md deleted file mode 100644 index 399885f0c1..0000000000 --- a/docs/cli/scaletest_workspace-traffic.md +++ /dev/null @@ -1,159 +0,0 @@ - - -# scaletest workspace-traffic - -Generate traffic to scaletest workspaces through coderd - -## Usage - -```console -coder scaletest workspace-traffic [flags] -``` - -## Options - -### --bytes-per-tick - -| | | -| ----------- | -------------------------------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_WORKSPACE_TRAFFIC_BYTES_PER_TICK | -| Default | 1024 | - -How much traffic to generate per tick. - -### --cleanup-concurrency - -| | | -| ----------- | ------------------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_CLEANUP_CONCURRENCY | -| Default | 1 | - -Number of concurrent cleanup jobs to run. 0 means unlimited. - -### --cleanup-job-timeout - -| | | -| ----------- | ------------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_JOB_TIMEOUT | -| Default | 5m | - -Timeout per job. Jobs may take longer to complete under higher concurrency limits. - -### --cleanup-timeout - -| | | -| ----------- | --------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_CLEANUP_TIMEOUT | -| Default | 30m | - -Timeout for the entire cleanup run. 0 means unlimited. - -### --concurrency - -| | | -| ----------- | ----------------------------------------- | -| Type | int | -| Environment | $CODER_SCALETEST_CONCURRENCY | -| Default | 1 | - -Number of concurrent jobs to run. 0 means unlimited. - -### --job-timeout - -| | | -| ----------- | ----------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_JOB_TIMEOUT | -| Default | 5m | - -Timeout per job. Jobs may take longer to complete under higher concurrency limits. - -### --output - -| | | -| ----------- | ------------------------------------- | -| Type | string-array | -| Environment | $CODER_SCALETEST_OUTPUTS | -| Default | text | - -Output format specs in the format "[:]". Not specifying a path will default to stdout. Available formats: text, json. - -### --scaletest-prometheus-address - -| | | -| ----------- | ------------------------------------------------ | -| Type | string | -| Environment | $CODER_SCALETEST_PROMETHEUS_ADDRESS | -| Default | 0.0.0.0:21112 | - -Address on which to expose scaletest Prometheus metrics. - -### --scaletest-prometheus-wait - -| | | -| ----------- | --------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_PROMETHEUS_WAIT | -| Default | 5s | - -How long to wait before exiting in order to allow Prometheus metrics to be scraped. - -### --tick-interval - -| | | -| ----------- | ------------------------------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_WORKSPACE_TRAFFIC_TICK_INTERVAL | -| Default | 100ms | - -How often to send traffic. - -### --timeout - -| | | -| ----------- | ------------------------------------- | -| Type | duration | -| Environment | $CODER_SCALETEST_TIMEOUT | -| Default | 30m | - -Timeout for the entire test run. 0 means unlimited. - -### --trace - -| | | -| ----------- | ----------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE | - -Whether application tracing data is collected. It exports to a backend configured by environment variables. See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md. - -### --trace-coder - -| | | -| ----------- | ----------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE_CODER | - -Whether opentelemetry traces are sent to Coder. We recommend keeping this disabled unless we advise you to enable it. - -### --trace-honeycomb-api-key - -| | | -| ----------- | ----------------------------------------------------- | -| Type | string | -| Environment | $CODER_SCALETEST_TRACE_HONEYCOMB_API_KEY | - -Enables trace exporting to Honeycomb.io using the provided API key. - -### --trace-propagate - -| | | -| ----------- | --------------------------------------------- | -| Type | bool | -| Environment | $CODER_SCALETEST_TRACE_PROPAGATE | - -Enables trace propagation to the Coder backend, which will be used to correlate server-side spans with client-side spans. Only enable this if the server is configured with the exact same tracing configuration as the client. diff --git a/docs/manifest.json b/docs/manifest.json index f759af4679..2299e59b39 100644 --- a/docs/manifest.json +++ b/docs/manifest.json @@ -632,26 +632,6 @@ "description": "Restart a workspace", "path": "cli/restart.md" }, - { - "title": "scaletest", - "description": "Run a scale test against the Coder API", - "path": "cli/scaletest.md" - }, - { - "title": "scaletest cleanup", - "description": "Cleanup scaletest workspaces, then cleanup scaletest users.", - "path": "cli/scaletest_cleanup.md" - }, - { - "title": "scaletest create-workspaces", - "description": "Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard.", - "path": "cli/scaletest_create-workspaces.md" - }, - { - "title": "scaletest workspace-traffic", - "description": "Generate traffic to scaletest workspaces through coderd", - "path": "cli/scaletest_workspace-traffic.md" - }, { "title": "schedule", "description": "Schedule automated start and stop times for workspaces", diff --git a/scaletest/lib/coder_workspacetraffic.sh b/scaletest/lib/coder_workspacetraffic.sh index 9c53db2a53..b9a5b53f86 100755 --- a/scaletest/lib/coder_workspacetraffic.sh +++ b/scaletest/lib/coder_workspacetraffic.sh @@ -80,7 +80,7 @@ spec: - command: - sh - -c - - "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${SCALETEST_TRAFFIC_BYTES_PER_TICK} --tick-interval=${SCALETEST_TRAFFIC_TICK_INTERVAL} --scaletest-prometheus-wait=60s" + - "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${SCALETEST_TRAFFIC_BYTES_PER_TICK} --tick-interval=${SCALETEST_TRAFFIC_TICK_INTERVAL} --scaletest-prometheus-wait=60s" env: - name: CODER_URL value: $CODER_URL diff --git a/scaletest/scaletest.sh b/scaletest/scaletest.sh index fdb23d9271..f14d8ef839 100755 --- a/scaletest/scaletest.sh +++ b/scaletest/scaletest.sh @@ -181,7 +181,7 @@ if [[ -n "${SCALETEST_CODER_LICENSE}" ]]; then fi echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces." -DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" scaletest create-workspaces \ +DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" exp scaletest create-workspaces \ --count "${SCALETEST_NUM_WORKSPACES}" \ --template=kubernetes \ --concurrency "${SCALETEST_CREATE_CONCURRENCY}" \