diff --git a/registry/coder/modules/git-clone/README.md b/registry/coder/modules/git-clone/README.md index 3336770f..54bb5f0f 100644 --- a/registry/coder/modules/git-clone/README.md +++ b/registry/coder/modules/git-clone/README.md @@ -14,7 +14,7 @@ This module allows you to automatically clone a repository by URL and skip if it module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" } @@ -28,7 +28,7 @@ module "git-clone" { module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" base_dir = "~/projects/coder" @@ -43,7 +43,7 @@ To use with [Git Authentication](https://coder.com/docs/v2/latest/admin/git-prov module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" } @@ -70,7 +70,7 @@ data "coder_parameter" "git_repo" { module "git_clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = data.coder_parameter.git_repo.value } @@ -105,7 +105,7 @@ Configuring `git-clone` for a self-hosted GitHub Enterprise Server running at `g module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.example.com/coder/coder/tree/feat/example" git_providers = { @@ -125,7 +125,7 @@ To GitLab clone with a specific branch like `feat/example` module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://gitlab.com/coder/coder/-/tree/feat/example" } @@ -137,7 +137,7 @@ Configuring `git-clone` for a self-hosted GitLab running at `gitlab.example.com` module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://gitlab.example.com/coder/coder/-/tree/feat/example" git_providers = { @@ -159,7 +159,7 @@ For example, to clone the `feat/example` branch: module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" branch_name = "feat/example" @@ -177,7 +177,7 @@ For example, this will clone into the `~/projects/coder/coder-dev` folder: module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" folder_name = "coder-dev" @@ -185,21 +185,32 @@ module "git-clone" { } ``` -## Git shallow clone +## Extra `git clone` arguments -Limit the clone history to speed-up workspace startup by setting `depth`. +> [!NOTE] +> **Upgrading from v1.x?** The `depth` variable was removed in v2.0.0. Use `extra_args = ["--depth=1"]` instead. +> Do not pass `-b` or `--branch` in `extra_args` when `branch_name` is +> already set (or extracted from the URL). Git silently accepts the last +> `-b` flag, so the two values would conflict. -When `depth` is greater than `0` the module runs `git clone --depth `. -If not defined, the default, `0`, performs a full clone. +Pass any additional flags through `extra_args` (one element per argument). +This lets you enable anything `git clone` supports without the module having +to expose it explicitly, for example a shallow clone, submodules, parallel +fetches, or partial clones. ```tf module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" - depth = 1 + extra_args = [ + "--depth=1", + "--recurse-submodules", + "--jobs=8", + "--filter=blob:none", + ] } ``` @@ -212,7 +223,7 @@ This is useful for preparing the environment or validating prerequisites before module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" pre_clone_script = <<-EOT @@ -235,7 +246,7 @@ This is useful for running initialization tasks like installing dependencies or module "git-clone" { count = data.coder_workspace.me.start_count source = "registry.coder.com/coder/git-clone/coder" - version = "1.3.1" + version = "2.0.0" agent_id = coder_agent.example.id url = "https://github.com/coder/coder" post_clone_script = <<-EOT @@ -248,3 +259,14 @@ module "git-clone" { EOT } ``` + +## Troubleshooting + +Clone output is logged to `~/.coder-modules/coder/git-clone//logs/clone.log`: + + cat ~/.coder-modules/coder/git-clone/*/logs/clone.log + +Pre-clone and post-clone script output is logged alongside: + + cat ~/.coder-modules/coder/git-clone/*/logs/pre_clone.log + cat ~/.coder-modules/coder/git-clone/*/logs/post_clone.log diff --git a/registry/coder/modules/git-clone/main.test.ts b/registry/coder/modules/git-clone/main.test.ts index af900eef..8ac2925b 100644 --- a/registry/coder/modules/git-clone/main.test.ts +++ b/registry/coder/modules/git-clone/main.test.ts @@ -1,11 +1,48 @@ import { describe, expect, it } from "bun:test"; import { - executeScriptInContainer, + execContainer, + findResourceInstance, + runContainer, runTerraformApply, runTerraformInit, testRequiredVariables, + type scriptOutput, + type TerraformState, } from "~test"; +const executeScriptInContainer = async ( + state: TerraformState, + image: string, + before?: string, +): Promise => { + const instance = findResourceInstance(state, "coder_script"); + const id = await runContainer(image); + await execContainer(id, ["sh", "-c", "apk add --no-cache bash >/dev/null"]); + if (before) { + await execContainer(id, ["sh", "-c", before]); + } + const resp = await execContainer(id, ["bash", "-c", instance.script]); + return { + exitCode: resp.exitCode, + stdout: resp.stdout.trim().split("\n"), + stderr: resp.stderr.trim().split("\n"), + }; +}; + +// Drops a fake `git` onto PATH that prints each argv entry on its own line. +// Lets tests prove that arguments (including ones with embedded spaces) reach +// `git clone` as single argv tokens, which the echo line cannot show because +// it joins with spaces. +const installFakeGit = [ + "cat > /usr/local/bin/git <<'SHIM'", + "#!/bin/sh", + 'for arg in "$@"; do', + ' printf "argv:%s\\n" "$arg"', + "done", + "SHIM", + "chmod +x /usr/local/bin/git", +].join("\n"); + describe("git-clone", async () => { await runTerraformInit(import.meta.dir); @@ -30,12 +67,11 @@ describe("git-clone", async () => { url: "fake-url", }); const output = await executeScriptInContainer(state, "alpine/git"); - expect(output.stdout).toEqual([ - "Creating directory ~/fake-url...", - "Cloning fake-url to ~/fake-url...", - ]); - expect(output.stderr.join(" ")).toContain("fatal"); - expect(output.stderr.join(" ")).toContain("fake-url"); + expect(output.stdout).toContain("Creating directory /root/fake-url..."); + expect(output.stdout).toContain("Cloning fake-url to /root/fake-url..."); + expect(output.exitCode).not.toBe(0); + expect(output.stdout.join(" ")).toContain("fatal"); + expect(output.stdout.join(" ")).toContain("fake-url"); }); it("repo_dir should match repo name for https", async () => { @@ -206,10 +242,12 @@ describe("git-clone", async () => { }); const output = await executeScriptInContainer(state, "alpine/git"); expect(output.exitCode).toBe(0); - expect(output.stdout).toEqual([ - "Creating directory ~/repo-tests.log...", - "Cloning https://github.com/michaelbrewer/repo-tests.log to ~/repo-tests.log on branch feat/branch...", - ]); + expect(output.stdout).toContain( + "Creating directory /root/repo-tests.log...", + ); + expect(output.stdout).toContain( + "Cloning https://github.com/michaelbrewer/repo-tests.log to /root/repo-tests.log on branch feat/branch...", + ); }); it("runs with gitlab clone with switch to feat/branch", async () => { @@ -219,10 +257,12 @@ describe("git-clone", async () => { }); const output = await executeScriptInContainer(state, "alpine/git"); expect(output.exitCode).toBe(0); - expect(output.stdout).toEqual([ - "Creating directory ~/repo-tests.log...", - "Cloning https://gitlab.com/mike.brew/repo-tests.log to ~/repo-tests.log on branch feat/branch...", - ]); + expect(output.stdout).toContain( + "Creating directory /root/repo-tests.log...", + ); + expect(output.stdout).toContain( + "Cloning https://gitlab.com/mike.brew/repo-tests.log to /root/repo-tests.log on branch feat/branch...", + ); }); it("runs with github clone with branch_name set to feat/branch", async () => { @@ -240,10 +280,12 @@ describe("git-clone", async () => { const output = await executeScriptInContainer(state, "alpine/git"); expect(output.exitCode).toBe(0); - expect(output.stdout).toEqual([ - "Creating directory ~/repo-tests.log...", - "Cloning https://github.com/michaelbrewer/repo-tests.log to ~/repo-tests.log on branch feat/branch...", - ]); + expect(output.stdout).toContain( + "Creating directory /root/repo-tests.log...", + ); + expect(output.stdout).toContain( + "Cloning https://github.com/michaelbrewer/repo-tests.log to /root/repo-tests.log on branch feat/branch...", + ); }); it("runs post-clone script", async () => { @@ -256,7 +298,6 @@ describe("git-clone", async () => { const output = await executeScriptInContainer( state, "alpine/git", - "sh", "mkdir -p /tmp/fake-url && echo 'existing' > /tmp/fake-url/file.txt", ); expect(output.stdout).toContain("Running post-clone script..."); @@ -272,7 +313,7 @@ describe("git-clone", async () => { const output = await executeScriptInContainer(state, "alpine/git"); expect(output.stdout).toContain("Running pre-clone script..."); expect(output.stdout).toContain("Pre-clone script executed"); - expect(output.stdout).toContain("Cloning fake-url to ~/fake-url..."); + expect(output.stdout).toContain("Cloning fake-url to /root/fake-url..."); }); it("fails when pre-clone script fails", async () => { @@ -285,7 +326,104 @@ describe("git-clone", async () => { expect(output.exitCode).toBe(42); expect(output.stdout).toContain("Running pre-clone script..."); expect(output.stdout).toContain("Pre-clone script failed"); - expect(output.stdout).not.toContain("Cloning fake-url to ~/fake-url..."); + expect(output.stdout).not.toContain( + "Cloning fake-url to /root/fake-url...", + ); + }); + + it("defaults extra_args to empty", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + url: "fake-url", + }); + const output = await executeScriptInContainer( + state, + "alpine/git", + installFakeGit, + ); + // With no extra_args the only argv tokens should be clone, url, path. + expect(output.stdout.join("\n")).toContain( + ["argv:clone", "argv:fake-url", "argv:/root/fake-url"].join("\n"), + ); + }); + + it("passes extra_args to git clone", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + url: "fake-url", + extra_args: JSON.stringify([ + "--recurse-submodules", + "--jobs=8", + "--config=user.name=Coder User", + "-c", + "core.sshCommand=ssh -i /tmp/key", + ]), + }); + const output = await executeScriptInContainer( + state, + "alpine/git", + installFakeGit, + ); + expect(output.exitCode).toBe(0); + expect(output.stdout.join("\n")).toContain( + [ + "argv:clone", + "argv:--recurse-submodules", + "argv:--jobs=8", + "argv:--config=user.name=Coder User", + "argv:-c", + "argv:core.sshCommand=ssh -i /tmp/key", + "argv:fake-url", + "argv:/root/fake-url", + ].join("\n"), + ); + }); + + it("passes extra_args alongside branch_name in the correct order", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + url: "fake-url", + branch_name: "feat/branch", + extra_args: JSON.stringify([ + "--recurse-submodules", + "--config=user.name=Coder User", + ]), + }); + const output = await executeScriptInContainer( + state, + "alpine/git", + installFakeGit, + ); + expect(output.exitCode).toBe(0); + expect(output.stdout.join("\n")).toContain( + [ + "argv:clone", + "argv:--recurse-submodules", + "argv:--config=user.name=Coder User", + "argv:-b", + "argv:feat/branch", + "argv:fake-url", + "argv:/root/fake-url", + ].join("\n"), + ); + }); + + it("writes output to logs/clone.log under module directory", async () => { + const state = await runTerraformApply(import.meta.dir, { + agent_id: "foo", + url: "fake-url", + }); + const instance = findResourceInstance(state, "coder_script"); + const id = await runContainer("alpine/git"); + await execContainer(id, ["sh", "-c", "apk add --no-cache bash >/dev/null"]); + await execContainer(id, ["bash", "-c", instance.script]); + const log = await execContainer(id, [ + "bash", + "-c", + "cat /root/.coder-modules/coder/git-clone/*/logs/clone.log", + ]); + expect(log.exitCode).toBe(0); + expect(log.stdout).toContain("Cloning fake-url to /root/fake-url..."); }); it("fails when post-clone script fails", async () => { @@ -298,7 +436,6 @@ describe("git-clone", async () => { const output = await executeScriptInContainer( state, "alpine/git", - "sh", "mkdir -p /tmp/fake-url && echo 'existing' > /tmp/fake-url/file.txt", ); expect(output.exitCode).toBe(43); diff --git a/registry/coder/modules/git-clone/main.tf b/registry/coder/modules/git-clone/main.tf index 1fb28a4d..b4e6a4f8 100644 --- a/registry/coder/modules/git-clone/main.tf +++ b/registry/coder/modules/git-clone/main.tf @@ -56,10 +56,10 @@ variable "folder_name" { default = "" } -variable "depth" { - description = "If > 0, perform a shallow clone using this depth." - type = number - default = 0 +variable "extra_args" { + description = "Extra arguments to pass to `git clone`, one element per argument (e.g. `[\"--recurse-submodules\", \"--jobs=8\", \"--filter=blob:none\"]`)." + type = list(string) + default = [] } variable "post_clone_script" { @@ -97,6 +97,30 @@ locals { encoded_post_clone_script = var.post_clone_script != null ? base64encode(var.post_clone_script) : "" # Encode the pre_clone_script for passing to the shell script encoded_pre_clone_script = var.pre_clone_script != null ? base64encode(var.pre_clone_script) : "" + encoded_extra_args = base64encode(join("\n", var.extra_args)) + + # Module directory paths (matches coder-utils convention) + # Use folder_name so two git-clone instances in the same template get + # separate script and log directories. + module_dir = "$HOME/.coder-modules/coder/git-clone/${local.folder_name}" + scripts_directory = "${local.module_dir}/scripts" + log_directory = "${local.module_dir}/logs" + clone_script_path = "${local.scripts_directory}/clone.sh" + clone_log_path = "${local.log_directory}/clone.log" + pre_clone_log_path = "${local.log_directory}/pre_clone.log" + post_clone_log_path = "${local.log_directory}/post_clone.log" + + encoded_clone_script = base64encode(templatefile("${path.module}/run.sh", { + CLONE_PATH = local.clone_path, + REPO_URL = local.clone_url, + BRANCH_NAME = local.branch_name, + EXTRA_ARGS = local.encoded_extra_args, + POST_CLONE_SCRIPT = local.encoded_post_clone_script, + PRE_CLONE_SCRIPT = local.encoded_pre_clone_script, + SCRIPTS_DIR = local.scripts_directory, + PRE_CLONE_LOG_PATH = local.pre_clone_log_path, + POST_CLONE_LOG_PATH = local.post_clone_log_path, + })) } output "repo_dir" { @@ -130,15 +154,21 @@ output "branch_name" { } resource "coder_script" "git_clone" { - agent_id = var.agent_id - script = templatefile("${path.module}/run.sh", { - CLONE_PATH = local.clone_path, - REPO_URL : local.clone_url, - BRANCH_NAME : local.branch_name, - DEPTH = var.depth, - POST_CLONE_SCRIPT : local.encoded_post_clone_script, - PRE_CLONE_SCRIPT : local.encoded_pre_clone_script, - }) + agent_id = var.agent_id + script = <<-EOT + #!/bin/bash + set -o errexit + set -o pipefail + + mkdir -p "${local.module_dir}" + mkdir -p "${local.scripts_directory}" + mkdir -p "${local.log_directory}" + + echo -n '${local.encoded_clone_script}' | base64 -d > "${local.clone_script_path}" + chmod +x "${local.clone_script_path}" + + "${local.clone_script_path}" 2>&1 | tee "${local.clone_log_path}" + EOT display_name = "Git Clone" icon = "/icon/git.svg" run_on_start = true diff --git a/registry/coder/modules/git-clone/run.sh b/registry/coder/modules/git-clone/run.sh index 76928a40..fb2456af 100644 --- a/registry/coder/modules/git-clone/run.sh +++ b/registry/coder/modules/git-clone/run.sh @@ -7,9 +7,12 @@ CLONE_PATH="${CLONE_PATH}" BRANCH_NAME="${BRANCH_NAME}" # Expand home if it's specified! CLONE_PATH="$${CLONE_PATH/#\~/$${HOME}}" -DEPTH="${DEPTH}" +EXTRA_ARGS="${EXTRA_ARGS}" POST_CLONE_SCRIPT="${POST_CLONE_SCRIPT}" PRE_CLONE_SCRIPT="${PRE_CLONE_SCRIPT}" +SCRIPTS_DIR="${SCRIPTS_DIR}" +PRE_CLONE_LOG_PATH="${PRE_CLONE_LOG_PATH}" +POST_CLONE_LOG_PATH="${POST_CLONE_LOG_PATH}" # Check if the variable is empty... if [ -z "$REPO_URL" ]; then @@ -39,11 +42,18 @@ fi # Run pre-clone script if provided if [ -n "$PRE_CLONE_SCRIPT" ]; then echo "Running pre-clone script..." - PRE_CLONE_TMP=$(mktemp) - echo "$PRE_CLONE_SCRIPT" | base64 -d > "$PRE_CLONE_TMP" - chmod +x "$PRE_CLONE_TMP" - $PRE_CLONE_TMP - rm "$PRE_CLONE_TMP" + PRE_CLONE_PATH="$SCRIPTS_DIR/pre_clone.sh" + echo "$PRE_CLONE_SCRIPT" | base64 -d > "$PRE_CLONE_PATH" + chmod +x "$PRE_CLONE_PATH" + "$PRE_CLONE_PATH" 2>&1 | tee "$PRE_CLONE_LOG_PATH" +fi + +# Build optional git clone flags +extra_args=() +if [ -n "$EXTRA_ARGS" ]; then + while IFS= read -r arg || [ -n "$arg" ]; do + [ -n "$arg" ] && extra_args+=("$arg") + done < <(echo "$EXTRA_ARGS" | base64 -d) fi # Check if the directory is empty @@ -51,18 +61,10 @@ fi if [ -z "$(ls -A "$CLONE_PATH")" ]; then if [ -z "$BRANCH_NAME" ]; then echo "Cloning $REPO_URL to $CLONE_PATH..." - if [ "$DEPTH" -gt 0 ]; then - git clone --depth "$DEPTH" "$REPO_URL" "$CLONE_PATH" - else - git clone "$REPO_URL" "$CLONE_PATH" - fi + git clone $${extra_args[@]+"$${extra_args[@]}"} "$REPO_URL" "$CLONE_PATH" else echo "Cloning $REPO_URL to $CLONE_PATH on branch $BRANCH_NAME..." - if [ "$DEPTH" -gt 0 ]; then - git clone --depth "$DEPTH" -b "$BRANCH_NAME" "$REPO_URL" "$CLONE_PATH" - else - git clone "$REPO_URL" -b "$BRANCH_NAME" "$CLONE_PATH" - fi + git clone $${extra_args[@]+"$${extra_args[@]}"} -b "$BRANCH_NAME" "$REPO_URL" "$CLONE_PATH" fi else echo "$CLONE_PATH already exists and isn't empty, skipping clone!" @@ -71,10 +73,9 @@ fi # Run post-clone script if provided if [ -n "$POST_CLONE_SCRIPT" ]; then echo "Running post-clone script..." - POST_CLONE_TMP=$(mktemp) - echo "$POST_CLONE_SCRIPT" | base64 -d > "$POST_CLONE_TMP" - chmod +x "$POST_CLONE_TMP" + POST_CLONE_PATH="$SCRIPTS_DIR/post_clone.sh" + echo "$POST_CLONE_SCRIPT" | base64 -d > "$POST_CLONE_PATH" + chmod +x "$POST_CLONE_PATH" cd "$CLONE_PATH" || exit - $POST_CLONE_TMP - rm "$POST_CLONE_TMP" + "$POST_CLONE_PATH" 2>&1 | tee "$POST_CLONE_LOG_PATH" fi