From 0008c135cf437d705c08c3789a70015efc06c7f2 Mon Sep 17 00:00:00 2001
From: Garrett Delfosse <garrett@coder.com>
Date: Mon, 13 Jan 2025 12:08:40 -0500
Subject: [PATCH] feat: add scaletest load generation infrastructure (#15816)

Closes https://github.com/coder/internal/issues/149

This creates workspaces in each region and runs traffic generation
against the workspaces colocated in the region.
---
 .../terraform/action/coder_helm_values.tftpl  |   8 +-
 scaletest/terraform/action/coder_templates.tf | 348 +++++++++++++-----
 scaletest/terraform/action/coder_traffic.tf   | 236 ++++++++++++
 .../terraform/action/coder_workspaces.tf      | 180 +++++++++
 scaletest/terraform/action/gcp_clusters.tf    |   5 +
 scaletest/terraform/action/k8s_coder_asia.tf  |   6 +
 .../terraform/action/k8s_coder_europe.tf      |   6 +
 .../terraform/action/k8s_coder_primary.tf     |   6 +-
 scaletest/terraform/action/kubeconfig.tftpl   |  17 +
 scaletest/terraform/action/scenarios.tf       |  95 +----
 10 files changed, 740 insertions(+), 167 deletions(-)
 create mode 100644 scaletest/terraform/action/coder_traffic.tf
 create mode 100644 scaletest/terraform/action/coder_workspaces.tf
 create mode 100644 scaletest/terraform/action/kubeconfig.tftpl

diff --git a/scaletest/terraform/action/coder_helm_values.tftpl b/scaletest/terraform/action/coder_helm_values.tftpl
index 7de0c598a1..be24bf61cd 100644
--- a/scaletest/terraform/action/coder_helm_values.tftpl
+++ b/scaletest/terraform/action/coder_helm_values.tftpl
@@ -34,7 +34,11 @@ coder:
     - name: "CODER_URL"
       value: "${access_url}"
     - name: "CODER_PROVISIONERD_TAGS"
-      value: "scope=organization"
+      value: "scope=organization,deployment=${deployment}"
+    - name: "CODER_PROVISIONER_DAEMON_NAME"
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
     - name: "CODER_CONFIG_DIR"
       value: "/tmp/config"
     %{~ endif ~}
@@ -76,6 +80,8 @@ coder:
       value: "${experiments}"
     - name: "CODER_DANGEROUS_DISABLE_RATE_LIMITS"
       value: "true"
+    - name: "CODER_DANGEROUS_ALLOW_PATH_APP_SITE_OWNER_ACCESS"
+      value: "true"
   image:
     repo: ${image_repo}
     tag: ${image_tag}
diff --git a/scaletest/terraform/action/coder_templates.tf b/scaletest/terraform/action/coder_templates.tf
index c2334a488a..d27c25844b 100644
--- a/scaletest/terraform/action/coder_templates.tf
+++ b/scaletest/terraform/action/coder_templates.tf
@@ -1,91 +1,111 @@
 resource "local_file" "kubernetes_template" {
   filename = "${path.module}/.coderv2/templates/kubernetes/main.tf"
   content  = <<EOF
-    terraform {
-      required_providers {
-        coder = {
-          source  = "coder/coder"
-          version = "~> 0.23.0"
-        }
-        kubernetes = {
-          source  = "hashicorp/kubernetes"
-          version = "~> 2.30"
-        }
-      }
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "~> 2.1.0"
     }
-
-    provider "coder" {}
-
-    provider "kubernetes" {
-      config_path = null # always use host
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.30"
     }
-
-    data "coder_workspace" "me" {}
-    data "coder_workspace_owner" "me" {}
-
-    resource "coder_agent" "main" {
-      os                     = "linux"
-      arch                   = "amd64"
-    }
-
-    resource "kubernetes_pod" "main" {
-      count = data.coder_workspace.me.start_count
-      metadata {
-        name      = "coder-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
-        namespace = "${local.coder_namespace}"
-        labels = {
-          "app.kubernetes.io/name"     = "coder-workspace"
-          "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
-        }
-      }
-      spec {
-        security_context {
-          run_as_user = "1000"
-          fs_group    = "1000"
-        }
-        container {
-          name              = "dev"
-          image             = "${var.workspace_image}"
-          image_pull_policy = "Always"
-          command           = ["sh", "-c", coder_agent.main.init_script]
-          security_context {
-            run_as_user = "1000"
-          }
-          env {
-            name  = "CODER_AGENT_TOKEN"
-            value = coder_agent.main.token
-          }
-          resources {
-            requests = {
-              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_request}"
-              "memory" = "${local.scenarios[var.scenario].workspaces.mem_request}"
-            }
-            limits = {
-              "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_limit}"
-              "memory" = "${local.scenarios[var.scenario].workspaces.mem_limit}"
-            }
-          }
-        }
-
-        affinity {
-          node_affinity {
-            required_during_scheduling_ignored_during_execution {
-              node_selector_term {
-                match_expressions {
-                  key = "cloud.google.com/gke-nodepool"
-                  operator = "In"
-                  values = ["${google_container_node_pool.node_pool["primary_workspaces"].name}","${google_container_node_pool.node_pool["europe_workspaces"].name}","${google_container_node_pool.node_pool["asia_workspaces"].name}"]
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  EOF
+  }
 }
 
-resource "kubernetes_config_map" "template" {
+provider "coder" {}
+
+provider "kubernetes" {
+  config_path = null # always use host
+}
+
+data "coder_workspace" "me" {}
+data "coder_workspace_owner" "me" {}
+
+resource "coder_agent" "main" {
+  os                     = "linux"
+  arch                   = "amd64"
+}
+
+resource "coder_script" "websocat" {
+  agent_id     = coder_agent.main.id
+  display_name = "websocat"
+  script       = <<EOF2
+curl -sSL -o /tmp/websocat https://github.com/vi/websocat/releases/download/v1.12.0/websocat.x86_64-unknown-linux-musl
+chmod +x /tmp/websocat
+
+/tmp/websocat --exit-on-eof --binary ws-l:127.0.0.1:1234 mirror:
+EOF2
+  run_on_start = true
+}
+
+resource "coder_app" "wsecho" {
+  agent_id     = coder_agent.main.id
+  slug         = "wsec"
+  display_name = "WebSocket Echo"
+  url          = "http://localhost:1234"
+  share        = "authenticated"
+}
+
+resource "kubernetes_pod" "main" {
+  count = data.coder_workspace.me.start_count
+  metadata {
+    name      = "coder-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+    namespace = "${local.coder_namespace}"
+    labels = {
+      "app.kubernetes.io/name"     = "coder-workspace"
+      "app.kubernetes.io/instance" = "coder-workspace-$${lower(data.coder_workspace_owner.me.name)}-$${lower(data.coder_workspace.me.name)}"
+    }
+  }
+  spec {
+    security_context {
+      run_as_user = "1000"
+      fs_group    = "1000"
+    }
+    container {
+      name              = "dev"
+      image             = "${var.workspace_image}"
+      image_pull_policy = "Always"
+      command           = ["sh", "-c", coder_agent.main.init_script]
+      security_context {
+        run_as_user = "1000"
+      }
+      env {
+        name  = "CODER_AGENT_TOKEN"
+        value = coder_agent.main.token
+      }
+      resources {
+        requests = {
+          "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_request}"
+          "memory" = "${local.scenarios[var.scenario].workspaces.mem_request}"
+        }
+        limits = {
+          "cpu"    = "${local.scenarios[var.scenario].workspaces.cpu_limit}"
+          "memory" = "${local.scenarios[var.scenario].workspaces.mem_limit}"
+        }
+      }
+    }
+
+    affinity {
+      node_affinity {
+        required_during_scheduling_ignored_during_execution {
+          node_selector_term {
+            match_expressions {
+              key = "cloud.google.com/gke-nodepool"
+              operator = "In"
+              values = ["${google_container_node_pool.node_pool["primary_workspaces"].name}","${google_container_node_pool.node_pool["europe_workspaces"].name}","${google_container_node_pool.node_pool["asia_workspaces"].name}"]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+EOF
+}
+
+resource "kubernetes_config_map" "template_primary" {
   provider = kubernetes.primary
 
   metadata {
@@ -98,7 +118,7 @@ resource "kubernetes_config_map" "template" {
   }
 }
 
-resource "kubernetes_job" "push_template" {
+resource "kubernetes_job" "push_template_primary" {
   provider = kubernetes.primary
 
   metadata {
@@ -137,8 +157,10 @@ resource "kubernetes_job" "push_template" {
             "templates",
             "push",
             "--directory=/home/coder/template",
+            "--provisioner-tag=scope=organization",
+            "--provisioner-tag=deployment=primary",
             "--yes",
-            "kubernetes"
+            "kubernetes-primary"
           ]
           volume_mount {
             name       = "coder-template"
@@ -149,7 +171,7 @@ resource "kubernetes_job" "push_template" {
         volume {
           name = "coder-template"
           config_map {
-            name = kubernetes_config_map.template.metadata.0.name
+            name = kubernetes_config_map.template_primary.metadata.0.name
           }
         }
         restart_policy = "Never"
@@ -157,4 +179,162 @@ resource "kubernetes_job" "push_template" {
     }
   }
   wait_for_completion = true
+
+  depends_on = [helm_release.provisionerd_primary]
+}
+
+resource "kubernetes_config_map" "template_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name      = "coder-template"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+  }
+
+  data = {
+    "main.tf" = local_file.kubernetes_template.content
+  }
+}
+
+resource "kubernetes_job" "push_template_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name      = "${var.name}-push-template"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-push-template"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["europe_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "templates",
+            "push",
+            "--directory=/home/coder/template",
+            "--provisioner-tag=scope=organization",
+            "--provisioner-tag=deployment=europe",
+            "--yes",
+            "kubernetes-europe"
+          ]
+          volume_mount {
+            name       = "coder-template"
+            mount_path = "/home/coder/template/main.tf"
+            sub_path   = "main.tf"
+          }
+        }
+        volume {
+          name = "coder-template"
+          config_map {
+            name = kubernetes_config_map.template_europe.metadata.0.name
+          }
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  depends_on = [helm_release.provisionerd_europe]
+}
+
+resource "kubernetes_config_map" "template_asia" {
+  provider = kubernetes.asia
+
+  metadata {
+    name      = "coder-template"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+  }
+
+  data = {
+    "main.tf" = local_file.kubernetes_template.content
+  }
+}
+
+resource "kubernetes_job" "push_template_asia" {
+  provider = kubernetes.asia
+
+  metadata {
+    name      = "${var.name}-push-template"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-push-template"
+    }
+  }
+  spec {
+    completions = 1
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["asia_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "templates",
+            "push",
+            "--directory=/home/coder/template",
+            "--provisioner-tag=scope=organization",
+            "--provisioner-tag=deployment=asia",
+            "--yes",
+            "kubernetes-asia"
+          ]
+          volume_mount {
+            name       = "coder-template"
+            mount_path = "/home/coder/template/main.tf"
+            sub_path   = "main.tf"
+          }
+        }
+        volume {
+          name = "coder-template"
+          config_map {
+            name = kubernetes_config_map.template_asia.metadata.0.name
+          }
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  depends_on = [helm_release.provisionerd_asia]
 }
diff --git a/scaletest/terraform/action/coder_traffic.tf b/scaletest/terraform/action/coder_traffic.tf
new file mode 100644
index 0000000000..bea829427a
--- /dev/null
+++ b/scaletest/terraform/action/coder_traffic.tf
@@ -0,0 +1,236 @@
+locals {
+  wait_baseline_duration = "5m"
+  bytes_per_tick         = 1024
+  tick_interval          = "100ms"
+
+  traffic_types = {
+    ssh = {
+      wait_duration = "0m"
+      duration      = "30m"
+      job_timeout   = "35m"
+      flags = [
+        "--ssh",
+      ]
+    }
+    webterminal = {
+      wait_duration = "5m"
+      duration      = "25m"
+      job_timeout   = "30m"
+      flags         = []
+    }
+    app = {
+      wait_duration = "10m"
+      duration      = "20m"
+      job_timeout   = "25m"
+      flags = [
+        "--app=wsec",
+      ]
+    }
+  }
+}
+
+resource "time_sleep" "wait_baseline" {
+  depends_on = [
+    kubernetes_job.create_workspaces_primary,
+    kubernetes_job.create_workspaces_europe,
+    kubernetes_job.create_workspaces_asia,
+  ]
+
+  create_duration = local.wait_baseline_duration
+}
+
+resource "time_sleep" "wait_traffic" {
+  for_each = local.traffic_types
+
+  depends_on = [time_sleep.wait_baseline]
+
+  create_duration = local.traffic_types[each.key].wait_duration
+}
+
+resource "kubernetes_job" "workspace_traffic_primary" {
+  provider = kubernetes.primary
+
+  for_each = local.traffic_types
+  metadata {
+    name      = "${var.name}-workspace-traffic-${each.key}"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-workspace-traffic-${each.key}"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = concat([
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "workspace-traffic",
+            "--template=kubernetes-primary",
+            "--concurrency=0",
+            "--bytes-per-tick=${local.bytes_per_tick}",
+            "--tick-interval=${local.tick_interval}",
+            "--scaletest-prometheus-wait=30s",
+            "--job-timeout=${local.traffic_types[each.key].duration}",
+          ], local.traffic_types[each.key].flags)
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.traffic_types[each.key].job_timeout
+  }
+
+  depends_on = [time_sleep.wait_baseline, time_sleep.wait_traffic[each.key]]
+}
+
+resource "kubernetes_job" "workspace_traffic_europe" {
+  provider = kubernetes.europe
+
+  for_each = local.traffic_types
+  metadata {
+    name      = "${var.name}-workspace-traffic-${each.key}"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-workspace-traffic-${each.key}"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["europe_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = concat([
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "workspace-traffic",
+            "--template=kubernetes-europe",
+            "--concurrency=0",
+            "--bytes-per-tick=${local.bytes_per_tick}",
+            "--tick-interval=${local.tick_interval}",
+            "--scaletest-prometheus-wait=30s",
+            "--job-timeout=${local.traffic_types[each.key].duration}",
+            "--workspace-proxy-url=${local.deployments.europe.url}",
+          ], local.traffic_types[each.key].flags)
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.traffic_types[each.key].job_timeout
+  }
+
+  depends_on = [time_sleep.wait_baseline, time_sleep.wait_traffic[each.key]]
+}
+
+resource "kubernetes_job" "workspace_traffic_asia" {
+  provider = kubernetes.asia
+
+  for_each = local.traffic_types
+  metadata {
+    name      = "${var.name}-workspace-traffic-${each.key}"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-workspace-traffic-${each.key}"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["asia_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = concat([
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "workspace-traffic",
+            "--template=kubernetes-asia",
+            "--concurrency=0",
+            "--bytes-per-tick=${local.bytes_per_tick}",
+            "--tick-interval=${local.tick_interval}",
+            "--scaletest-prometheus-wait=30s",
+            "--job-timeout=${local.traffic_types[each.key].duration}",
+            "--workspace-proxy-url=${local.deployments.asia.url}",
+          ], local.traffic_types[each.key].flags)
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.traffic_types[each.key].job_timeout
+  }
+
+  depends_on = [time_sleep.wait_baseline, time_sleep.wait_traffic[each.key]]
+}
diff --git a/scaletest/terraform/action/coder_workspaces.tf b/scaletest/terraform/action/coder_workspaces.tf
new file mode 100644
index 0000000000..f49c1c9968
--- /dev/null
+++ b/scaletest/terraform/action/coder_workspaces.tf
@@ -0,0 +1,180 @@
+locals {
+  create_workspace_timeout = "30m"
+}
+
+resource "kubernetes_job" "create_workspaces_primary" {
+  provider = kubernetes.primary
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_primary.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["primary_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-primary",
+            "--concurrency=${local.scenarios[var.scenario].provisionerd.replicas}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_primary]
+}
+
+resource "kubernetes_job" "create_workspaces_europe" {
+  provider = kubernetes.europe
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_europe.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["europe_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-europe",
+            "--concurrency=${local.scenarios[var.scenario].provisionerd.replicas}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_europe]
+}
+
+resource "kubernetes_job" "create_workspaces_asia" {
+  provider = kubernetes.asia
+
+  metadata {
+    name      = "${var.name}-create-workspaces"
+    namespace = kubernetes_namespace.coder_asia.metadata.0.name
+    labels = {
+      "app.kubernetes.io/name" = "${var.name}-create-workspaces"
+    }
+  }
+  spec {
+    completions   = 1
+    backoff_limit = 0
+    template {
+      metadata {}
+      spec {
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "cloud.google.com/gke-nodepool"
+                  operator = "In"
+                  values   = ["${google_container_node_pool.node_pool["asia_misc"].name}"]
+                }
+              }
+            }
+          }
+        }
+        container {
+          name  = "cli"
+          image = "${var.coder_image_repo}:${var.coder_image_tag}"
+          command = [
+            "/opt/coder",
+            "--verbose",
+            "--url=${local.deployments.primary.url}",
+            "--token=${trimspace(data.local_file.api_key.content)}",
+            "exp",
+            "scaletest",
+            "create-workspaces",
+            "--count=${local.scenarios[var.scenario].workspaces.count_per_deployment}",
+            "--template=kubernetes-asia",
+            "--concurrency=${local.scenarios[var.scenario].provisionerd.replicas}",
+            "--no-cleanup"
+          ]
+        }
+        restart_policy = "Never"
+      }
+    }
+  }
+  wait_for_completion = true
+
+  timeouts {
+    create = local.create_workspace_timeout
+  }
+
+  depends_on = [kubernetes_job.push_template_asia]
+}
diff --git a/scaletest/terraform/action/gcp_clusters.tf b/scaletest/terraform/action/gcp_clusters.tf
index 5ca3bac3f7..c41d06c6c1 100644
--- a/scaletest/terraform/action/gcp_clusters.tf
+++ b/scaletest/terraform/action/gcp_clusters.tf
@@ -139,6 +139,11 @@ resource "google_container_node_pool" "node_pool" {
     metadata = {
       disable-legacy-endpoints = "true"
     }
+    kubelet_config {
+      cpu_manager_policy = ""
+      cpu_cfs_quota      = false
+      pod_pids_limit     = 0
+    }
   }
   lifecycle {
     ignore_changes = [management[0].auto_repair, management[0].auto_upgrade, timeouts]
diff --git a/scaletest/terraform/action/k8s_coder_asia.tf b/scaletest/terraform/action/k8s_coder_asia.tf
index 2d22173498..307a50136e 100644
--- a/scaletest/terraform/action/k8s_coder_asia.tf
+++ b/scaletest/terraform/action/k8s_coder_asia.tf
@@ -70,7 +70,10 @@ resource "helm_release" "coder_asia" {
     mem_request      = local.scenarios[var.scenario].coder.mem_request,
     cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
+    deployment       = "asia",
   })]
+
+  depends_on = [null_resource.license]
 }
 
 resource "helm_release" "provisionerd_asia" {
@@ -100,5 +103,8 @@ resource "helm_release" "provisionerd_asia" {
     mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
     cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
+    deployment       = "asia",
   })]
+
+  depends_on = [null_resource.license]
 }
diff --git a/scaletest/terraform/action/k8s_coder_europe.tf b/scaletest/terraform/action/k8s_coder_europe.tf
index bb6140aef7..b6169c84a5 100644
--- a/scaletest/terraform/action/k8s_coder_europe.tf
+++ b/scaletest/terraform/action/k8s_coder_europe.tf
@@ -70,7 +70,10 @@ resource "helm_release" "coder_europe" {
     mem_request      = local.scenarios[var.scenario].coder.mem_request,
     cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
+    deployment       = "europe",
   })]
+
+  depends_on = [null_resource.license]
 }
 
 resource "helm_release" "provisionerd_europe" {
@@ -100,5 +103,8 @@ resource "helm_release" "provisionerd_europe" {
     mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
     cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
+    deployment       = "europe",
   })]
+
+  depends_on = [null_resource.license]
 }
diff --git a/scaletest/terraform/action/k8s_coder_primary.tf b/scaletest/terraform/action/k8s_coder_primary.tf
index 68dc7fc80a..0c4a64815a 100644
--- a/scaletest/terraform/action/k8s_coder_primary.tf
+++ b/scaletest/terraform/action/k8s_coder_primary.tf
@@ -90,10 +90,11 @@ resource "helm_release" "coder_primary" {
     mem_request      = local.scenarios[var.scenario].coder.mem_request,
     cpu_limit        = local.scenarios[var.scenario].coder.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].coder.mem_limit,
+    deployment       = "primary",
   })]
 }
 
-resource "helm_release" "provisionerd_chart" {
+resource "helm_release" "provisionerd_primary" {
   provider = helm.primary
 
   repository = local.coder_helm_repo
@@ -120,5 +121,8 @@ resource "helm_release" "provisionerd_chart" {
     mem_request      = local.scenarios[var.scenario].provisionerd.mem_request,
     cpu_limit        = local.scenarios[var.scenario].provisionerd.cpu_limit,
     mem_limit        = local.scenarios[var.scenario].provisionerd.mem_limit,
+    deployment       = "primary",
   })]
+
+  depends_on = [null_resource.license]
 }
diff --git a/scaletest/terraform/action/kubeconfig.tftpl b/scaletest/terraform/action/kubeconfig.tftpl
new file mode 100644
index 0000000000..d997edf456
--- /dev/null
+++ b/scaletest/terraform/action/kubeconfig.tftpl
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+current-context: ${name}
+clusters:
+- name: ${name}
+  cluster:
+    certificate-authority-data: ${cluster_ca_certificate}
+    server: ${endpoint}
+contexts:
+- context:
+    cluster: ${name}
+    user: ${name}
+  name: ${name}
+users:
+- name: ${name}
+  user:
+    token: ${access_token}
diff --git a/scaletest/terraform/action/scenarios.tf b/scaletest/terraform/action/scenarios.tf
index 996c449d28..bd22fa7c5b 100644
--- a/scaletest/terraform/action/scenarios.tf
+++ b/scaletest/terraform/action/scenarios.tf
@@ -1,101 +1,34 @@
 locals {
   scenarios = {
-    small = {
-      coder = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-        replicas      = 1
-        cpu_request   = "1000m"
-        mem_request   = "6Gi"
-        cpu_limit     = "2000m"
-        mem_limit     = "12Gi"
-      }
-      provisionerd = {
-        replicas    = 1
-        cpu_request = "100m"
-        mem_request = "1Gi"
-        cpu_limit   = "1000m"
-        mem_limit   = "1Gi"
-      }
-      workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
-      }
-      misc = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-      }
-      cloudsql = {
-        tier            = "db-f1-micro"
-        max_connections = 500
-      }
-    }
-    medium = {
-      coder = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-8"
-        replicas      = 1
-        cpu_request   = "3000m"
-        mem_request   = "12Gi"
-        cpu_limit     = "6000m"
-        mem_limit     = "24Gi"
-      }
-      provisionerd = {
-        replicas    = 1
-        cpu_request = "100m"
-        mem_request = "1Gi"
-        cpu_limit   = "1000m"
-        mem_limit   = "1Gi"
-      }
-      workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-8"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
-      }
-      misc = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-4"
-      }
-      cloudsql = {
-        tier            = "db-custom-1-3840"
-        max_connections = 500
-      }
-    }
     large = {
       coder = {
         nodepool_size = 3
-        machine_type  = "t2d-standard-8"
+        machine_type  = "c2d-standard-8"
         replicas      = 3
-        cpu_request   = "1000m"
-        mem_request   = "6Gi"
-        cpu_limit     = "2000m"
+        cpu_request   = "4000m"
+        mem_request   = "12Gi"
+        cpu_limit     = "4000m"
         mem_limit     = "12Gi"
       }
       provisionerd = {
-        replicas    = 1
+        replicas    = 30
         cpu_request = "100m"
-        mem_request = "1Gi"
+        mem_request = "256Mi"
         cpu_limit   = "1000m"
         mem_limit   = "1Gi"
       }
       workspaces = {
-        nodepool_size = 1
-        machine_type  = "t2d-standard-8"
-        cpu_request   = "100m"
-        mem_request   = "128Mi"
-        cpu_limit     = "100m"
-        mem_limit     = "128Mi"
+        count_per_deployment = 100
+        nodepool_size        = 3
+        machine_type         = "c2d-standard-32"
+        cpu_request          = "100m"
+        mem_request          = "128Mi"
+        cpu_limit            = "100m"
+        mem_limit            = "128Mi"
       }
       misc = {
         nodepool_size = 1
-        machine_type  = "t2d-standard-4"
+        machine_type  = "c2d-standard-32"
       }
       cloudsql = {
         tier            = "db-custom-2-7680"