From 289f0217c7d0ee0ecb8692b772ef9d3052eebe92 Mon Sep 17 00:00:00 2001 From: Spike Curtis Date: Thu, 25 Sep 2025 16:18:37 +0400 Subject: [PATCH] feat: add scaletest Runner for dynamicparameters load gen (#19890) relates to https://github.com/coder/internal/issues/912 Adds a new scaletest Runner to generate dynamic parameters load. A later PR will add the CLI command, including creating the template & version. --- .github/workflows/typos.toml | 1 + coderd/coderdtest/dynamicparameters.go | 15 ++- provisioner/echo/serve.go | 18 +++ scaletest/dynamicparameters/config.go | 10 ++ scaletest/dynamicparameters/metrics.go | 28 ++++ scaletest/dynamicparameters/run.go | 114 +++++++++++++++++ scaletest/dynamicparameters/run_test.go | 49 +++++++ scaletest/dynamicparameters/template.go | 74 +++++++++++ scaletest/dynamicparameters/tf/main.tf | 120 ++++++++++++++++++ .../dynamicparameters/tf/modules/two/main.tf | 31 +++++ 10 files changed, 458 insertions(+), 2 deletions(-) create mode 100644 scaletest/dynamicparameters/config.go create mode 100644 scaletest/dynamicparameters/metrics.go create mode 100644 scaletest/dynamicparameters/run.go create mode 100644 scaletest/dynamicparameters/run_test.go create mode 100644 scaletest/dynamicparameters/template.go create mode 100644 scaletest/dynamicparameters/tf/main.tf create mode 100644 scaletest/dynamicparameters/tf/modules/two/main.tf diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index 6f47566811..8b3f77c1ef 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -1,5 +1,6 @@ [default] extend-ignore-identifiers-re = ["gho_.*"] +extend-ignore-re = ["(#|//)\\s*spellchecker:ignore-next-line\\n.*"] [default.extend-identifiers] alog = "alog" diff --git a/coderd/coderdtest/dynamicparameters.go b/coderd/coderdtest/dynamicparameters.go index c6adb6c97e..1cb60632ae 100644 --- a/coderd/coderdtest/dynamicparameters.go +++ b/coderd/coderdtest/dynamicparameters.go @@ -20,6 +20,9 @@ type DynamicParameterTemplateParams struct { Plan json.RawMessage ModulesArchive []byte + // ExtraFiles are additional files to include in the template, beyond the MainTF. + ExtraFiles map[string][]byte + // Uses a zip archive instead of a tar Zip bool @@ -36,9 +39,17 @@ type DynamicParameterTemplateParams struct { func DynamicParameterTemplate(t *testing.T, client *codersdk.Client, org uuid.UUID, args DynamicParameterTemplateParams) (codersdk.Template, codersdk.TemplateVersion) { t.Helper() - files := echo.WithExtraFiles(map[string][]byte{ + // Start with main.tf + extraFiles := map[string][]byte{ "main.tf": []byte(args.MainTF), - }) + } + + // Add any additional files + for name, content := range args.ExtraFiles { + extraFiles[name] = content + } + + files := echo.WithExtraFiles(extraFiles) files.ProvisionPlan = []*proto.Response{{ Type: &proto.Response_Plan{ Plan: &proto.PlanComplete{ diff --git a/provisioner/echo/serve.go b/provisioner/echo/serve.go index 4bb2a1dd6b..5069424156 100644 --- a/provisioner/echo/serve.go +++ b/provisioner/echo/serve.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "text/template" @@ -359,9 +360,26 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response } } } + dirs := []string{} for name, content := range responses.ExtraFiles { logger.Debug(ctx, "extra file", slog.F("name", name)) + // We need to add directories before any files that use them. But, we only need to do this + // once. + dir := filepath.Dir(name) + if dir != "." && !slices.Contains(dirs, dir) { + logger.Debug(ctx, "adding extra file directory", slog.F("dir", dir)) + dirs = append(dirs, dir) + err := writer.WriteHeader(&tar.Header{ + Name: dir, + Mode: 0o755, + Typeflag: tar.TypeDir, + }) + if err != nil { + return nil, err + } + } + err := writer.WriteHeader(&tar.Header{ Name: name, Size: int64(len(content)), diff --git a/scaletest/dynamicparameters/config.go b/scaletest/dynamicparameters/config.go new file mode 100644 index 0000000000..176b1245b2 --- /dev/null +++ b/scaletest/dynamicparameters/config.go @@ -0,0 +1,10 @@ +package dynamicparameters + +import "github.com/google/uuid" + +type Config struct { + TemplateVersion uuid.UUID `json:"template_version"` + SessionToken string `json:"session_token"` + Metrics *Metrics `json:"-"` + MetricLabelValues []string `json:"metric_label_values"` +} diff --git a/scaletest/dynamicparameters/metrics.go b/scaletest/dynamicparameters/metrics.go new file mode 100644 index 0000000000..cd2c689977 --- /dev/null +++ b/scaletest/dynamicparameters/metrics.go @@ -0,0 +1,28 @@ +package dynamicparameters + +import "github.com/prometheus/client_golang/prometheus" + +type Metrics struct { + LatencyInitialResponseSeconds prometheus.HistogramVec + LatencyChangeResponseSeconds prometheus.HistogramVec +} + +func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics { + m := &Metrics{ + LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "dynamic_parameters_latency_initial_response_seconds", + Help: "Time in seconds to get the initial dynamic parameters response from start of request.", + }, labelNames), + LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "dynamic_parameters_latency_change_response_seconds", + Help: "Time in seconds to between sending a dynamic parameters change request and receiving the response.", + }, labelNames), + } + reg.MustRegister(m.LatencyInitialResponseSeconds) + reg.MustRegister(m.LatencyChangeResponseSeconds) + return m +} diff --git a/scaletest/dynamicparameters/run.go b/scaletest/dynamicparameters/run.go new file mode 100644 index 0000000000..cec482bb41 --- /dev/null +++ b/scaletest/dynamicparameters/run.go @@ -0,0 +1,114 @@ +package dynamicparameters + +import ( + "context" + "fmt" + "io" + "slices" + "time" + + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/scaletest/harness" + "github.com/coder/websocket" +) + +type Runner struct { + client *codersdk.Client + cfg Config +} + +var _ harness.Runnable = &Runner{} + +func NewRunner(client *codersdk.Client, cfg Config) *Runner { + clone := codersdk.New(client.URL) + clone.HTTPClient = client.HTTPClient + clone.SetLogger(client.Logger()) + clone.SetSessionToken(cfg.SessionToken) + return &Runner{ + client: clone, + cfg: cfg, + } +} + +// Run executes the dynamic parameters test, which: +// +// 1. connects to the dynamic parameters stream +// 2. waits for the initial response +// 3. sends a change request +// 4. waits for the change response +// 5. closes the stream +func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) { + startTime := time.Now() + stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion) + if err != nil { + return xerrors.Errorf("connect to dynamic parameters stream: %w", err) + } + defer stream.Close(websocket.StatusNormalClosure) + respCh := stream.Chan() + + var initTime time.Time + select { + case <-ctx.Done(): + return ctx.Err() + case resp, ok := <-respCh: + if !ok { + return xerrors.Errorf("dynamic parameters stream closed before initial response") + } + initTime = time.Now() + r.cfg.Metrics.LatencyInitialResponseSeconds. + WithLabelValues(r.cfg.MetricLabelValues...). + Observe(initTime.Sub(startTime).Seconds()) + _, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp) + if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool { + return p.Name == "zero" + }) { + return xerrors.Errorf("missing expected parameter: 'zero'") + } + if err := checkNoDiagnostics(resp); err != nil { + return xerrors.Errorf("unexpected initial response diagnostics: %w", err) + } + } + + err = stream.Send(codersdk.DynamicParametersRequest{ + ID: 1, + Inputs: map[string]string{ + "zero": "B", + }, + }) + if err != nil { + return xerrors.Errorf("send change request: %w", err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case resp, ok := <-respCh: + if !ok { + return xerrors.Errorf("dynamic parameters stream closed before change response") + } + _, _ = fmt.Fprintf(logs, "change response: %+v\n", resp) + r.cfg.Metrics.LatencyChangeResponseSeconds. + WithLabelValues(r.cfg.MetricLabelValues...). + Observe(time.Since(initTime).Seconds()) + if resp.ID != 1 { + return xerrors.Errorf("unexpected response ID: %d", resp.ID) + } + if err := checkNoDiagnostics(resp); err != nil { + return xerrors.Errorf("unexpected change response diagnostics: %w", err) + } + return nil + } +} + +func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error { + if len(resp.Diagnostics) != 0 { + return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics) + } + for _, param := range resp.Parameters { + if len(param.Diagnostics) != 0 { + return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics) + } + } + return nil +} diff --git a/scaletest/dynamicparameters/run_test.go b/scaletest/dynamicparameters/run_test.go new file mode 100644 index 0000000000..57577d2743 --- /dev/null +++ b/scaletest/dynamicparameters/run_test.go @@ -0,0 +1,49 @@ +package dynamicparameters_test + +import ( + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + + "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/scaletest/dynamicparameters" + "github.com/coder/coder/v2/testutil" +) + +func TestRun(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitLong) + + client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) + client.SetLogger(testutil.Logger(t).Leveled(slog.LevelDebug)) + first := coderdtest.CreateFirstUser(t, client) + userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID) + orgID := first.OrganizationID + + dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent() + require.NoError(t, err) + + template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{ + MainTF: dynamicParametersTerraformSource, + Plan: nil, + ModulesArchive: nil, + StaticParams: nil, + ExtraFiles: dynamicparameters.GetModuleFiles(), + }) + + reg := prometheus.NewRegistry() + cfg := dynamicparameters.Config{ + TemplateVersion: version.ID, + SessionToken: userClient.SessionToken(), + Metrics: dynamicparameters.NewMetrics(reg, "template", "test_label_name"), + MetricLabelValues: []string{template.Name, "test_label_value"}, + } + runner := dynamicparameters.NewRunner(userClient, cfg) + var logs strings.Builder + err = runner.Run(ctx, t.Name(), &logs) + t.Log("Runner logs:\n\n" + logs.String()) + require.NoError(t, err) +} diff --git a/scaletest/dynamicparameters/template.go b/scaletest/dynamicparameters/template.go new file mode 100644 index 0000000000..464fba2f7f --- /dev/null +++ b/scaletest/dynamicparameters/template.go @@ -0,0 +1,74 @@ +package dynamicparameters + +import ( + _ "embed" + "encoding/json" + "strings" + "text/template" + + "github.com/coder/coder/v2/cryptorand" +) + +//go:embed tf/main.tf +var templateContent string + +func TemplateContent() (string, error) { + randomString, err := cryptorand.String(8) + if err != nil { + return "", err + } + tmpl, err := template.New("workspace-template").Parse(templateContent) + if err != nil { + return "", err + } + var result strings.Builder + err = tmpl.Execute(&result, map[string]string{ + "RandomString": randomString, + }) + if err != nil { + return "", err + } + return result.String(), nil +} + +//go:embed tf/modules/two/main.tf +var moduleTwoMainTF string + +// GetModuleFiles returns a map of module files to be used with ExtraFiles +func GetModuleFiles() map[string][]byte { + // Create the modules.json that Terraform needs to see the module + modulesJSON := struct { + Modules []struct { + Key string `json:"Key"` + Source string `json:"Source"` + Dir string `json:"Dir"` + } `json:"Modules"` + }{ + Modules: []struct { + Key string `json:"Key"` + Source string `json:"Source"` + Dir string `json:"Dir"` + }{ + { + Key: "", + Source: "", + Dir: ".", + }, + { + Key: "two", + Source: "./modules/two", + Dir: "modules/two", + }, + }, + } + + modulesJSONBytes, err := json.Marshal(modulesJSON) + if err != nil { + panic(err) // This should never happen with static data + } + + return map[string][]byte{ + "modules/two/main.tf": []byte(moduleTwoMainTF), + ".terraform/modules/modules.json": modulesJSONBytes, + } +} diff --git a/scaletest/dynamicparameters/tf/main.tf b/scaletest/dynamicparameters/tf/main.tf new file mode 100644 index 0000000000..64d0aa8abf --- /dev/null +++ b/scaletest/dynamicparameters/tf/main.tf @@ -0,0 +1,120 @@ +# Cache busting string so each copy of the template is unique: {{.RandomString}} +terraform { + required_providers { + coder = { + source = "coder/coder" + version = "2.5.3" + } + } +} + +locals { + one_options = { + "A" = ["AA", "AB"] + # spellchecker:ignore-next-line + "B" = ["BA", "BB"] + } + + three_options = { + "AA" = ["AAA", "AAB"] + "AB" = ["ABA", "ABB"] + # spellchecker:ignore-next-line + "BA" = ["BAA", "BAB"] + "BB" = ["BBA", "BBB"] + } + + username = data.coder_workspace_owner.me.name +} + +data "coder_workspace_owner" "me" {} + +data "coder_parameter" "zero" { + name = "zero" + display_name = "Root" + description = "Hello ${local.username}, pick your next parameter using this `dropdown` parameter." + form_type = "dropdown" + mutable = true + default = "A" + + option { + value = "A" + name = "A" + } + + option { + value = "B" + name = "B" + } +} + +data "coder_parameter" "one" { + + name = "One" + display_name = "Level One" + description = "This is the first level." + + type = "list(string)" + form_type = "multi-select" + order = 2 + mutable = true + default = "[\"${local.one_options[data.coder_parameter.zero.value][0]}\"]" + + dynamic "option" { + for_each = local.one_options[data.coder_parameter.zero.value] + content { + name = option.value + value = option.value + } + } +} + +module "two" { + source = "./modules/two" + + one_value = data.coder_parameter.one.value +} + +data "coder_parameter" "three" { + + name = "Three" + display_name = "Level Three" + description = "This is the third level." + + type = "string" + form_type = "radio" + order = 4 + mutable = true + default = local.three_options[module.two.two_value][0] + + dynamic "option" { + for_each = local.three_options[module.two.two_value] + content { + name = option.value + value = option.value + } + } +} + +data "coder_parameter" "four" { + name = "four" + display_name = "Level Four" + description = "This is the last level." + order = 5 + + type = "string" + form_type = "radio" + default = "a_fake_value_to_satisfy_import" + + option { + name = format("%s-%s", local.username, data.coder_parameter.three.value) + value = "a_fake_value_to_satisfy_import" + } + + dynamic "option" { + for_each = data.coder_workspace_owner.me.rbac_roles + content { + name = format("%s-%s", option.value.name, data.coder_parameter.three.value) + value = option.value.name + } + } +} diff --git a/scaletest/dynamicparameters/tf/modules/two/main.tf b/scaletest/dynamicparameters/tf/modules/two/main.tf new file mode 100644 index 0000000000..ba3d166d78 --- /dev/null +++ b/scaletest/dynamicparameters/tf/modules/two/main.tf @@ -0,0 +1,31 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + version = "2.5.3" + } + } +} + +variable "one_value" { + description = "The value from the 'one' parameter" + type = string +} + +data "coder_parameter" "two" { + name = "Two" + display_name = "Level Two" + description = "This is the second level." + + type = "string" + form_type = "textarea" + order = 3 + mutable = true + + default = trim(var.one_value, "[\"]") +} + +output "two_value" { + description = "The value of the 'two' parameter" + value = data.coder_parameter.two.value +}