shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: track terraform module source type in telemetry (#15590)

Browse Source
This commit is contained in:
Hugo Dutka
2024-11-20 11:03:48 +01:00
committed by GitHub
parent fbe2fa66f5
commit 97ce44a77d
2 changed files with 170 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/telemetry/telemetry.go
+89
View File
@@ -11,6 +11,7 @@ import (
"net/http"
"net/url"
"os"
"regexp"
"runtime"
"slices"
"strings"
@@ -680,9 +681,95 @@ func shouldSendRawModuleSource(source string) bool {
return strings.Contains(source, "registry.coder.com")
}
// ModuleSourceType is the type of source for a module.
// For reference, see https://developer.hashicorp.com/terraform/language/modules/sources
type ModuleSourceType string
const (
ModuleSourceTypeLocal ModuleSourceType = "local"
ModuleSourceTypeLocalAbs ModuleSourceType = "local_absolute"
ModuleSourceTypePublicRegistry ModuleSourceType = "public_registry"
ModuleSourceTypePrivateRegistry ModuleSourceType = "private_registry"
ModuleSourceTypeCoderRegistry ModuleSourceType = "coder_registry"
ModuleSourceTypeGitHub ModuleSourceType = "github"
ModuleSourceTypeBitbucket ModuleSourceType = "bitbucket"
ModuleSourceTypeGit ModuleSourceType = "git"
ModuleSourceTypeMercurial ModuleSourceType = "mercurial"
ModuleSourceTypeHTTP ModuleSourceType = "http"
ModuleSourceTypeS3 ModuleSourceType = "s3"
ModuleSourceTypeGCS ModuleSourceType = "gcs"
ModuleSourceTypeUnknown ModuleSourceType = "unknown"
)
// Terraform supports a variety of module source types, like:
// - local paths (./ or ../)
// - absolute local paths (/)
// - git URLs (git:: or git@)
// - http URLs
// - s3 URLs
//
// and more!
//
// See https://developer.hashicorp.com/terraform/language/modules/sources for an overview.
//
// This function attempts to classify the source type of a module. It's imperfect,
// as checks that terraform actually does are pretty complicated.
// See e.g. https://github.com/hashicorp/go-getter/blob/842d6c379e5e70d23905b8f6b5a25a80290acb66/detect.go#L47
// if you're interested in the complexity.
func GetModuleSourceType(source string) ModuleSourceType {
source = strings.TrimSpace(source)
source = strings.ToLower(source)
if strings.HasPrefix(source, "./") || strings.HasPrefix(source, "../") {
return ModuleSourceTypeLocal
}
if strings.HasPrefix(source, "/") {
return ModuleSourceTypeLocalAbs
}
// Match public registry modules in the format <NAMESPACE>/<NAME>/<PROVIDER>
// Sources can have a `//...` suffix, which signifies a subdirectory.
// The allowed characters are based on
// https://developer.hashicorp.com/terraform/cloud-docs/api-docs/private-registry/modules#request-body-1
// because Hashicorp's documentation about module sources doesn't mention it.
if matched, _ := regexp.MatchString(`^[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+(//.*)?$`, source); matched {
return ModuleSourceTypePublicRegistry
}
if strings.Contains(source, "github.com") {
return ModuleSourceTypeGitHub
}
if strings.Contains(source, "bitbucket.org") {
return ModuleSourceTypeBitbucket
}
if strings.HasPrefix(source, "git::") || strings.HasPrefix(source, "git@") {
return ModuleSourceTypeGit
}
if strings.HasPrefix(source, "hg::") {
return ModuleSourceTypeMercurial
}
if strings.HasPrefix(source, "http://") || strings.HasPrefix(source, "https://") {
return ModuleSourceTypeHTTP
}
if strings.HasPrefix(source, "s3::") {
return ModuleSourceTypeS3
}
if strings.HasPrefix(source, "gcs::") {
return ModuleSourceTypeGCS
}
if strings.Contains(source, "registry.terraform.io") {
return ModuleSourceTypePublicRegistry
}
if strings.Contains(source, "app.terraform.io") || strings.Contains(source, "localterraform.com") {
return ModuleSourceTypePrivateRegistry
}
if strings.Contains(source, "registry.coder.com") {
return ModuleSourceTypeCoderRegistry
}
return ModuleSourceTypeUnknown
}
func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
source := module.Source
version := module.Version
sourceType := GetModuleSourceType(source)
if !shouldSendRawModuleSource(source) {
source = fmt.Sprintf("%x", sha256.Sum256([]byte(source)))
version = fmt.Sprintf("%x", sha256.Sum256([]byte(version)))
@@ -694,6 +781,7 @@ func ConvertWorkspaceModule(module database.WorkspaceModule) WorkspaceModule {
Transition: module.Transition,
Source: source,
Version: version,
SourceType: sourceType,
Key: module.Key,
CreatedAt: module.CreatedAt,
}
@@ -938,6 +1026,7 @@ type WorkspaceModule struct {
Key string `json:"key"`
Version string `json:"version"`
Source string `json:"source"`
SourceType ModuleSourceType `json:"source_type"`
}
type WorkspaceAgent struct {
coderd/telemetry/telemetry_test.go
+81 -2
View File
@@ -133,7 +133,7 @@ func TestTelemetry(t *testing.T) {
})
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
JobID: pj.ID,
Source: "internal-url.com/some-module",
Source: "https://internal-url.com/some-module",
Version: "1.0.0",
})
_, snapshot := collectSnapshot(t, db, nil)
@@ -142,10 +142,89 @@ func TestTelemetry(t *testing.T) {
sort.Slice(modules, func(i, j int) bool {
return modules[i].Source < modules[j].Source
})
require.Equal(t, modules[0].Source, "921c61d6f3eef5118f3cae658d1518b378c5b02a4955a766c791440894d989c5")
require.Equal(t, modules[0].Source, "ed662ec0396db67e77119f14afcb9253574cc925b04a51d4374bcb1eae299f5d")
require.Equal(t, modules[0].Version, "92521fc3cbd964bdc9f584a991b89fddaa5754ed1cc96d6d42445338669c1305")
require.Equal(t, modules[0].SourceType, telemetry.ModuleSourceTypeHTTP)
require.Equal(t, modules[1].Source, "registry.coder.com/terraform/aws")
require.Equal(t, modules[1].Version, "1.0.0")
require.Equal(t, modules[1].SourceType, telemetry.ModuleSourceTypeCoderRegistry)
})
t.Run("ModuleSourceType", func(t *testing.T) {
t.Parallel()
cases := []struct {
source string
want telemetry.ModuleSourceType
}{
// Local relative paths
{source: "./modules/terraform-aws-vpc", want: telemetry.ModuleSourceTypeLocal},
{source: "../shared/modules/vpc", want: telemetry.ModuleSourceTypeLocal},
{source: " ./my-module ", want: telemetry.ModuleSourceTypeLocal}, // with whitespace
// Local absolute paths
{source: "/opt/terraform/modules/vpc", want: telemetry.ModuleSourceTypeLocalAbs},
{source: "/Users/dev/modules/app", want: telemetry.ModuleSourceTypeLocalAbs},
{source: "/etc/terraform/modules/network", want: telemetry.ModuleSourceTypeLocalAbs},
// Public registry
{source: "hashicorp/consul/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "registry.terraform.io/hashicorp/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "terraform-aws-modules/vpc/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "hashicorp/consul/aws//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "hashicorp/co-nsul/aw_s//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
// Private registry
{source: "app.terraform.io/company/vpc/aws", want: telemetry.ModuleSourceTypePrivateRegistry},
{source: "localterraform.com/org/module", want: telemetry.ModuleSourceTypePrivateRegistry},
{source: "APP.TERRAFORM.IO/test/module", want: telemetry.ModuleSourceTypePrivateRegistry}, // case insensitive
// Coder registry
{source: "registry.coder.com/terraform/aws", want: telemetry.ModuleSourceTypeCoderRegistry},
{source: "registry.coder.com/modules/base", want: telemetry.ModuleSourceTypeCoderRegistry},
{source: "REGISTRY.CODER.COM/test/module", want: telemetry.ModuleSourceTypeCoderRegistry}, // case insensitive
// GitHub
{source: "github.com/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeGitHub},
{source: "git::https://github.com/org/repo.git", want: telemetry.ModuleSourceTypeGitHub},
{source: "git::https://github.com/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeGitHub},
// Bitbucket
{source: "bitbucket.org/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeBitbucket},
{source: "git::https://bitbucket.org/org/repo.git", want: telemetry.ModuleSourceTypeBitbucket},
{source: "https://bitbucket.org/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeBitbucket},
// Generic Git
{source: "git::ssh://git.internal.com/repo.git", want: telemetry.ModuleSourceTypeGit},
{source: "git@gitlab.com:org/repo.git", want: telemetry.ModuleSourceTypeGit},
{source: "git::https://git.internal.com/repo.git?ref=v1.0.0", want: telemetry.ModuleSourceTypeGit},
// Mercurial
{source: "hg::https://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
{source: "hg::http://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
{source: "hg::ssh://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
// HTTP
{source: "https://example.com/vpc-module.zip", want: telemetry.ModuleSourceTypeHTTP},
{source: "http://example.com/modules/vpc", want: telemetry.ModuleSourceTypeHTTP},
{source: "https://internal.network/terraform/modules", want: telemetry.ModuleSourceTypeHTTP},
// S3
{source: "s3::https://s3-eu-west-1.amazonaws.com/bucket/vpc", want: telemetry.ModuleSourceTypeS3},
{source: "s3::https://bucket.s3.amazonaws.com/vpc", want: telemetry.ModuleSourceTypeS3},
{source: "s3::http://bucket.s3.amazonaws.com/vpc?version=1", want: telemetry.ModuleSourceTypeS3},
// GCS
{source: "gcs::https://www.googleapis.com/storage/v1/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
{source: "gcs::https://storage.googleapis.com/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
{source: "gcs::https://bucket.storage.googleapis.com/vpc", want: telemetry.ModuleSourceTypeGCS},
// Unknown
{source: "custom://example.com/vpc", want: telemetry.ModuleSourceTypeUnknown},
{source: "something-random", want: telemetry.ModuleSourceTypeUnknown},
{source: "", want: telemetry.ModuleSourceTypeUnknown},
}
for _, c := range cases {
require.Equal(t, c.want, telemetry.GetModuleSourceType(c.source))
}
})
}