shishantbiswas/coder
1
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-06-02 20:48:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: remove dbmem (#18803)

Browse Source 
Remove the in-memory database. Addresses #15109.
This commit is contained in:
Hugo Dutka
2025-07-09 09:46:31 +02:00
committed by GitHub
parent 1319ae293f
commit 3c2f3d640b
33 changed files with 146 additions and 15003 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.claude/docs/DATABASE.md
-39
View File
@@ -58,31 +58,6 @@ If adding fields to auditable types:
- `ActionSecret`: Field contains sensitive data
3. Run `make gen` to verify no audit errors
## In-Memory Database (dbmem) Updates
### Critical Requirements
When adding new fields to database structs:
- **CRITICAL**: Update `coderd/database/dbmem/dbmem.go` in-memory implementations
- The `Insert*` functions must include ALL new fields, not just basic ones
- Common issue: Tests pass with real database but fail with in-memory database due to missing field mappings
- Always verify in-memory database functions match the real database schema after migrations
### Example Pattern
```go
// In dbmem.go - ensure ALL fields are included
code := database.OAuth2ProviderAppCode{
ID: arg.ID,
CreatedAt: arg.CreatedAt,
// ... existing fields ...
ResourceUri: arg.ResourceUri, // New field
CodeChallenge: arg.CodeChallenge, // New field
CodeChallengeMethod: arg.CodeChallengeMethod, // New field
}
```
## Database Architecture
### Core Components
@@ -116,7 +91,6 @@ roles, err := db.GetAuthorizationUserRoles(dbauthz.AsSystemRestricted(ctx), user
1. **Nullable field errors**: Use `sql.Null*` types consistently
2. **Missing audit entries**: Update `enterprise/audit/table.go`
3. **dbmem inconsistencies**: Ensure in-memory implementations match schema
### Query Issues
@@ -139,19 +113,6 @@ func TestDatabaseFunction(t *testing.T) {
}
```
### In-Memory Testing
```go
func TestInMemoryDatabase(t *testing.T) {
db := dbmem.New()
// Test with in-memory database
result, err := db.GetSomething(ctx, param)
require.NoError(t, err)
require.Equal(t, expected, result)
}
```
## Best Practices
### Schema Design
.claude/docs/OAUTH2.md
+7 -8
View File
@@ -112,14 +112,13 @@ Always run the full test suite after OAuth2 changes:
## Common OAuth2 Issues
1. **OAuth2 endpoints returning wrong error format** - Ensure OAuth2 endpoints return RFC 6749 compliant errors
2. **OAuth2 tests failing but scripts working** - Check in-memory database implementations in `dbmem.go`
3. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
4. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
5. **RFC compliance failures** - Verify against actual RFC specifications, not assumptions
6. **Authorization context errors in public endpoints** - Use `dbauthz.AsSystemRestricted(ctx)` pattern
7. **Default value mismatches** - Ensure database migrations match application code defaults
8. **Bearer token authentication issues** - Check token extraction precedence and format validation
9. **URI validation failures** - Support both standard schemes and custom schemes per protocol requirements
2. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
3. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
4. **RFC compliance failures** - Verify against actual RFC specifications, not assumptions
5. **Authorization context errors in public endpoints** - Use `dbauthz.AsSystemRestricted(ctx)` pattern
6. **Default value mismatches** - Ensure database migrations match application code defaults
7. **Bearer token authentication issues** - Check token extraction precedence and format validation
8. **URI validation failures** - Support both standard schemes and custom schemes per protocol requirements
## Authorization Context Patterns
.claude/docs/TESTING.md
+4 -31
View File
@@ -39,31 +39,6 @@
2. **Verify information disclosure protections**
3. **Test token security and proper invalidation**
## Database Testing
### In-Memory Database Testing
When adding new database fields:
- **CRITICAL**: Update `coderd/database/dbmem/dbmem.go` in-memory implementations
- The `Insert*` functions must include ALL new fields, not just basic ones
- Common issue: Tests pass with real database but fail with in-memory database due to missing field mappings
- Always verify in-memory database functions match the real database schema after migrations
Example pattern:
```go
// In dbmem.go - ensure ALL fields are included
code := database.OAuth2ProviderAppCode{
ID: arg.ID,
CreatedAt: arg.CreatedAt,
// ... existing fields ...
ResourceUri: arg.ResourceUri, // New field
CodeChallenge: arg.CodeChallenge, // New field
CodeChallengeMethod: arg.CodeChallengeMethod, // New field
}
```
## Test Organization
### Test File Structure
@@ -107,15 +82,13 @@ coderd/
### Database-Related
1. **Tests passing locally but failing in CI** - Check if `dbmem` implementation needs updating
2. **SQL type errors** - Use `sql.Null*` types for nullable fields
3. **Race conditions in tests** - Use unique identifiers instead of hardcoded names
1. **SQL type errors** - Use `sql.Null*` types for nullable fields
2. **Race conditions in tests** - Use unique identifiers instead of hardcoded names
### OAuth2 Testing
1. **OAuth2 tests failing but scripts working** - Check in-memory database implementations in `dbmem.go`
2. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
3. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
1. **PKCE tests failing** - Verify both authorization code storage and token exchange handle PKCE fields
2. **Resource indicator validation failing** - Ensure database stores and retrieves resource parameters correctly
### General Issues
.claude/docs/TROUBLESHOOTING.md
+12 -21
View File
@@ -21,59 +21,50 @@
}
```
3. **Tests passing locally but failing in CI**
- **Solution**: Check if `dbmem` implementation needs updating
- Update `coderd/database/dbmem/dbmem.go` for Insert/Update methods
- Missing fields in dbmem can cause tests to fail even if main implementation is correct
### Testing Issues
4. **"package should be X_test"**
3. **"package should be X_test"**
- **Solution**: Use `package_test` naming for test files
- Example: `identityprovider_test` for black-box testing
5. **Race conditions in tests**
4. **Race conditions in tests**
- **Solution**: Use unique identifiers instead of hardcoded names
- Example: `fmt.Sprintf("test-client-%s-%d", t.Name(), time.Now().UnixNano())`
- Never use hardcoded names in concurrent tests
6. **Missing newlines**
5. **Missing newlines**
- **Solution**: Ensure files end with newline character
- Most editors can be configured to add this automatically
### OAuth2 Issues
7. **OAuth2 endpoints returning wrong error format**
6. **OAuth2 endpoints returning wrong error format**
- **Solution**: Ensure OAuth2 endpoints return RFC 6749 compliant errors
- Use standard error codes: `invalid_client`, `invalid_grant`, `invalid_request`
- Format: `{"error": "code", "error_description": "details"}`
8. **OAuth2 tests failing but scripts working**
- **Solution**: Check in-memory database implementations in `dbmem.go`
- Ensure all OAuth2 fields are properly copied in Insert/Update methods
9. **Resource indicator validation failing**
7. **Resource indicator validation failing**
- **Solution**: Ensure database stores and retrieves resource parameters correctly
- Check both authorization code storage and token exchange handling
10. **PKCE tests failing**
8. **PKCE tests failing**
- **Solution**: Verify both authorization code storage and token exchange handle PKCE fields
- Check `CodeChallenge` and `CodeChallengeMethod` field handling
### RFC Compliance Issues
11. **RFC compliance failures**
9. **RFC compliance failures**
- **Solution**: Verify against actual RFC specifications, not assumptions
- Use WebFetch tool to get current RFC content for compliance verification
- Read the actual RFC specifications before implementation
12. **Default value mismatches**
10. **Default value mismatches**
- **Solution**: Ensure database migrations match application code defaults
- Example: RFC 7591 specifies `client_secret_basic` as default, not `client_secret_post`
### Authorization Issues
13. **Authorization context errors in public endpoints**
11. **Authorization context errors in public endpoints**
- **Solution**: Use `dbauthz.AsSystemRestricted(ctx)` pattern
- Example:
@@ -84,17 +75,17 @@
### Authentication Issues
14. **Bearer token authentication issues**
12. **Bearer token authentication issues**
- **Solution**: Check token extraction precedence and format validation
- Ensure proper RFC 6750 Bearer Token Support implementation
15. **URI validation failures**
13. **URI validation failures**
- **Solution**: Support both standard schemes and custom schemes per protocol requirements
- Native OAuth2 apps may use custom schemes
### General Development Issues
16. **Log message formatting errors**
14. **Log message formatting errors**
- **Solution**: Use lowercase, descriptive messages without special characters
- Follow Go logging conventions
.claude/docs/WORKFLOWS.md
+2 -8
View File
@@ -81,11 +81,6 @@
- Add each new field with appropriate action (ActionTrack, ActionIgnore, ActionSecret)
- Run `make gen` to verify no audit errors
6. **In-memory database (dbmem) updates**:
- When adding new fields to database structs, ensure `dbmem` implementation copies all fields
- Check `coderd/database/dbmem/dbmem.go` for Insert/Update methods
- Missing fields in dbmem can cause tests to fail even if main implementation is correct
### Database Generation Process
1. Modify SQL files in `coderd/database/queries/`
@@ -164,9 +159,8 @@
1. **Development server won't start** - Use `./scripts/develop.sh` instead of manual commands
2. **Database migration errors** - Check migration file format and use helper scripts
3. **Test failures after database changes** - Update `dbmem` implementations
4. **Audit table errors** - Update `enterprise/audit/table.go` with new fields
5. **OAuth2 compliance issues** - Ensure RFC-compliant error responses
3. **Audit table errors** - Update `enterprise/audit/table.go` with new fields
4. **OAuth2 compliance issues** - Ensure RFC-compliant error responses
### Debug Commands
.github/workflows/ci.yaml
+2 -143
View File
@@ -311,94 +311,6 @@ jobs:
- name: Check for unstaged files
run: ./scripts/check_unstaged.sh
test-go:
runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-2022
steps:
- name: Harden Runner
# Harden Runner is only supported on Ubuntu runners.
if: runner.os == 'Linux'
uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2
with:
egress-policy: audit
# Set up RAM disks to speed up the rest of the job. This action is in
# a separate repository to allow its use before actions/checkout.
- name: Setup RAM Disks
if: runner.os == 'Windows'
uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go Paths
uses: ./.github/actions/setup-go-paths
- name: Setup Go
uses: ./.github/actions/setup-go
with:
# Runners have Go baked-in and Go will automatically
# download the toolchain configured in go.mod, so we don't
# need to reinstall it. It's faster on Windows runners.
use-preinstalled-go: ${{ runner.os == 'Windows' }}
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Download Test Cache
id: download-cache
uses: ./.github/actions/test-cache/download
with:
key-prefix: test-go-${{ runner.os }}-${{ runner.arch }}
- name: Test with Mock Database
id: test
shell: bash
run: |
# if macOS, install google-chrome for scaletests. As another concern,
# should we really have this kind of external dependency requirement
# on standard CI?
if [ "${{ matrix.os }}" == "macos-latest" ]; then
brew install google-chrome
fi
# By default Go will use the number of logical CPUs, which
# is a fine default.
PARALLEL_FLAG=""
# macOS will output "The default interactive shell is now zsh"
# intermittently in CI...
if [ "${{ matrix.os }}" == "macos-latest" ]; then
touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
fi
export TS_DEBUG_DISCO=true
gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" --rerun-fails=2 \
--packages="./..." -- $PARALLEL_FLAG -short
- name: Upload Test Cache
uses: ./.github/actions/test-cache/upload
with:
cache-key: ${{ steps.download-cache.outputs.cache-key }}
- name: Upload test stats to Datadog
timeout-minutes: 1
continue-on-error: true
uses: ./.github/actions/upload-datadog
if: success() || failure()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
test-go-pg:
# make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
# when changing runner sizes
@@ -567,7 +479,7 @@ jobs:
# We rerun failing tests to counteract flakiness coming from Postgres
# choking on macOS and Windows sometimes.
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
--format standard-quiet --packages "./..." \
-- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
@@ -655,55 +567,6 @@ jobs:
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
test-go-race:
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
timeout-minutes: 25
steps:
- name: Harden Runner
uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2
with:
egress-policy: audit
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Download Test Cache
id: download-cache
uses: ./.github/actions/test-cache/download
with:
key-prefix: test-go-race-${{ runner.os }}-${{ runner.arch }}
# We run race tests with reduced parallelism because they use more CPU and we were finding
# instances where tests appear to hang for multiple seconds, resulting in flaky tests when
# short timeouts are used.
# c.f. discussion on https://github.com/coder/coder/pull/15106
- name: Run Tests
run: |
gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
- name: Upload Test Cache
uses: ./.github/actions/test-cache/upload
with:
cache-key: ${{ steps.download-cache.outputs.cache-key }}
- name: Upload test stats to Datadog
timeout-minutes: 1
continue-on-error: true
uses: ./.github/actions/upload-datadog
if: always()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
test-go-race-pg:
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }}
needs: changes
@@ -741,7 +604,7 @@ jobs:
POSTGRES_VERSION: "17"
run: |
make test-postgres-docker
DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
gotestsum --junitfile="gotests.xml" --packages="./..." --rerun-fails=2 --rerun-fails-abort-on-data-race -- -race -parallel 4 -p 4
- name: Upload Test Cache
uses: ./.github/actions/test-cache/upload
@@ -1037,9 +900,7 @@ jobs:
- fmt
- lint
- gen
- test-go
- test-go-pg
- test-go-race
- test-go-race-pg
- test-js
- test-e2e
@@ -1060,9 +921,7 @@ jobs:
echo "- fmt: ${{ needs.fmt.result }}"
echo "- lint: ${{ needs.lint.result }}"
echo "- gen: ${{ needs.gen.result }}"
echo "- test-go: ${{ needs.test-go.result }}"
echo "- test-go-pg: ${{ needs.test-go-pg.result }}"
echo "- test-go-race: ${{ needs.test-go-race.result }}"
echo "- test-go-race-pg: ${{ needs.test-go-race-pg.result }}"
echo "- test-js: ${{ needs.test-js.result }}"
echo "- test-e2e: ${{ needs.test-e2e.result }}"
.golangci.yaml
-1
View File
@@ -181,7 +181,6 @@ linters-settings:
issues:
exclude-dirs:
- coderd/database/dbmem
- node_modules
- .git
CLAUDE.md
+2 -4
View File
@@ -44,7 +44,6 @@
2. Run `make gen`
3. If audit errors: update `enterprise/audit/table.go`
4. Run `make gen` again
5. Update `coderd/database/dbmem/dbmem.go` in-memory implementations
### LSP Navigation (USE FIRST)
@@ -116,9 +115,8 @@ app, err := api.Database.GetOAuth2ProviderAppByClientID(ctx, clientID)
1. **Audit table errors** → Update `enterprise/audit/table.go`
2. **OAuth2 errors** → Return RFC-compliant format
3. **dbmem failures** → Update in-memory implementations
4. **Race conditions** → Use unique test identifiers
5. **Missing newlines** → Ensure files end with newline
3. **Race conditions** → Use unique test identifiers
4. **Missing newlines** → Ensure files end with newline
---
Makefile
+1 -2
View File
@@ -599,7 +599,6 @@ DB_GEN_FILES := \
coderd/database/dump.sql \
coderd/database/querier.go \
coderd/database/unique_constraint.go \
coderd/database/dbmem/dbmem.go \
coderd/database/dbmetrics/dbmetrics.go \
coderd/database/dbauthz/dbauthz.go \
coderd/database/dbmock/dbmock.go
@@ -973,7 +972,7 @@ sqlc-vet: test-postgres-docker
test-postgres: test-postgres-docker
# The postgres test is prone to failure, so we limit parallelism for
# more consistent execution.
$(GIT_FLAGS) DB=ci gotestsum \
$(GIT_FLAGS) gotestsum \
--junitfile="gotests.xml" \
--jsonfile="gotests.json" \
$(GOTESTSUM_RETRY_FLAGS) \
cli/server.go
+30 -36
View File
@@ -77,7 +77,6 @@ import (
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/awsiamrds"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/dbmetrics"
"github.com/coder/coder/v2/coderd/database/dbpurge"
"github.com/coder/coder/v2/coderd/database/migrations"
@@ -423,7 +422,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
builtinPostgres := false
// Only use built-in if PostgreSQL URL isn't specified!
if !vals.InMemoryDatabase && vals.PostgresURL == "" {
if vals.PostgresURL == "" {
var closeFunc func() error
cliui.Infof(inv.Stdout, "Using built-in PostgreSQL (%s)", config.PostgresPath())
customPostgresCacheDir := ""
@@ -726,42 +725,37 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
// nil, that case of the select will just never fire, but it's important not to have a
// "bare" read on this channel.
var pubsubWatchdogTimeout <-chan struct{}
if vals.InMemoryDatabase {
// This is only used for testing.
options.Database = dbmem.New()
options.Pubsub = pubsub.NewInMemory()
} else {
sqlDB, dbURL, err := getAndMigratePostgresDB(ctx, logger, vals.PostgresURL.String(), codersdk.PostgresAuth(vals.PostgresAuth), sqlDriver)
if err != nil {
return xerrors.Errorf("connect to postgres: %w", err)
}
defer func() {
_ = sqlDB.Close()
}()
if options.DeploymentValues.Prometheus.Enable {
// At this stage we don't think the database name serves much purpose in these metrics.
// It requires parsing the DSN to determine it, which requires pulling in another dependency
// (i.e. https://github.com/jackc/pgx), but it's rather heavy.
// The conn string (https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) can
// take different forms, which make parsing non-trivial.
options.PrometheusRegistry.MustRegister(collectors.NewDBStatsCollector(sqlDB, ""))
}
options.Database = database.New(sqlDB)
ps, err := pubsub.New(ctx, logger.Named("pubsub"), sqlDB, dbURL)
if err != nil {
return xerrors.Errorf("create pubsub: %w", err)
}
options.Pubsub = ps
if options.DeploymentValues.Prometheus.Enable {
options.PrometheusRegistry.MustRegister(ps)
}
defer options.Pubsub.Close()
psWatchdog := pubsub.NewWatchdog(ctx, logger.Named("pswatch"), ps)
pubsubWatchdogTimeout = psWatchdog.Timeout()
defer psWatchdog.Close()
sqlDB, dbURL, err := getAndMigratePostgresDB(ctx, logger, vals.PostgresURL.String(), codersdk.PostgresAuth(vals.PostgresAuth), sqlDriver)
if err != nil {
return xerrors.Errorf("connect to postgres: %w", err)
}
defer func() {
_ = sqlDB.Close()
}()
if options.DeploymentValues.Prometheus.Enable {
// At this stage we don't think the database name serves much purpose in these metrics.
// It requires parsing the DSN to determine it, which requires pulling in another dependency
// (i.e. https://github.com/jackc/pgx), but it's rather heavy.
// The conn string (https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) can
// take different forms, which make parsing non-trivial.
options.PrometheusRegistry.MustRegister(collectors.NewDBStatsCollector(sqlDB, ""))
}
options.Database = database.New(sqlDB)
ps, err := pubsub.New(ctx, logger.Named("pubsub"), sqlDB, dbURL)
if err != nil {
return xerrors.Errorf("create pubsub: %w", err)
}
options.Pubsub = ps
if options.DeploymentValues.Prometheus.Enable {
options.PrometheusRegistry.MustRegister(ps)
}
defer options.Pubsub.Close()
psWatchdog := pubsub.NewWatchdog(ctx, logger.Named("pswatch"), ps)
pubsubWatchdogTimeout = psWatchdog.Timeout()
defer psWatchdog.Close()
if options.DeploymentValues.Prometheus.Enable && options.DeploymentValues.Prometheus.CollectDBMetrics {
options.Database = dbmetrics.NewQueryMetrics(options.Database, options.Logger, options.PrometheusRegistry)
cli/server_test.go
-3
View File
@@ -59,9 +59,6 @@ import (
)
func dbArg(t *testing.T) string {
if !dbtestutil.WillUsePostgres() {
return "--in-memory"
}
dbURL, err := dbtestutil.Open(t)
require.NoError(t, err)
return "--postgres-url=" + dbURL
cli/testdata/server-config.yaml.golden
Vendored
-3
View File
@@ -462,9 +462,6 @@ enableSwagger: false
# to be configured as a shared directory across coderd/provisionerd replicas.
# (default: [cache dir], type: string)
cacheDir: [cache dir]
# Controls whether data will be stored in an in-memory database.
# (default: <unset>, type: bool)
inMemoryDatabase: false
# Controls whether Coder data, including built-in Postgres, will be stored in a
# temporary directory and deleted when the server is stopped.
# (default: <unset>, type: bool)
coderd/apidoc/docs.go
Generated
-3
View File
@@ -12305,9 +12305,6 @@ const docTemplate = `{
"http_cookies": {
"$ref": "#/definitions/codersdk.HTTPCookieConfig"
},
"in_memory_database": {
"type": "boolean"
},
"job_hang_detector_interval": {
"type": "integer"
},
coderd/apidoc/swagger.json
Generated
-3
View File
@@ -10989,9 +10989,6 @@
"http_cookies": {
"$ref": "#/definitions/codersdk.HTTPCookieConfig"
},
"in_memory_database": {
"type": "boolean"
},
"job_hang_detector_interval": {
"type": "integer"
},
coderd/database/dbauthz/customroles_test.go
+2 -2
View File
@@ -12,7 +12,7 @@ import (
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/db2sdk"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/codersdk"
@@ -202,7 +202,7 @@ func TestInsertCustomRoles(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
db := dbmem.New()
db, _ := dbtestutil.NewDB(t)
rec := &coderdtest.RecordingAuthorizer{
Wrapped: rbac.NewAuthorizer(prometheus.NewRegistry()),
}
coderd/database/dbauthz/dbauthz_test.go
+36 -75
View File
@@ -19,7 +19,6 @@ import (
"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/database/db2sdk"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/notifications"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/codersdk"
@@ -3661,148 +3660,119 @@ func (s *MethodTestSuite) TestExtraMethods() {
func (s *MethodTestSuite) TestTailnetFunctions() {
s.Run("CleanTailnetCoordinators", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("CleanTailnetLostPeers", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("CleanTailnetTunnels", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("DeleteAllTailnetClientSubscriptions", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteAllTailnetClientSubscriptionsParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("DeleteAllTailnetTunnels", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteAllTailnetTunnelsParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("DeleteCoordinator", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("DeleteTailnetAgent", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteTailnetAgentParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).Errors(sql.ErrNoRows).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).Errors(sql.ErrNoRows)
}))
s.Run("DeleteTailnetClient", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteTailnetClientParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).Errors(sql.ErrNoRows).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).Errors(sql.ErrNoRows)
}))
s.Run("DeleteTailnetClientSubscription", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteTailnetClientSubscriptionParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete)
}))
s.Run("DeleteTailnetPeer", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteTailnetPeerParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented).
ErrorsWithPG(sql.ErrNoRows)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).Errors(sql.ErrNoRows)
}))
s.Run("DeleteTailnetTunnel", s.Subtest(func(_ database.Store, check *expects) {
check.Args(database.DeleteTailnetTunnelParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).
ErrorsWithInMemDB(dbmem.ErrUnimplemented).
ErrorsWithPG(sql.ErrNoRows)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionDelete).Errors(sql.ErrNoRows)
}))
s.Run("GetAllTailnetAgents", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetTailnetAgents", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetTailnetClientsForAgent", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetTailnetPeers", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetTailnetTunnelPeerBindings", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetTailnetTunnelPeerIDs", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetAllTailnetCoordinators", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetAllTailnetPeers", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("GetAllTailnetTunnels", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionRead)
}))
s.Run("UpsertTailnetAgent", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpsertTailnetAgentParams{Node: json.RawMessage("{}")}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate)
}))
s.Run("UpsertTailnetClient", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpsertTailnetClientParams{Node: json.RawMessage("{}")}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate)
}))
s.Run("UpsertTailnetClientSubscription", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpsertTailnetClientSubscriptionParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate)
}))
s.Run("UpsertTailnetCoordinator", s.Subtest(func(_ database.Store, check *expects) {
check.Args(uuid.New()).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate)
}))
s.Run("UpsertTailnetPeer", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpsertTailnetPeerParams{
Status: database.TailnetStatusOk,
}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionCreate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionCreate)
}))
s.Run("UpsertTailnetTunnel", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpsertTailnetTunnelParams{}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionCreate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionCreate)
}))
s.Run("UpdateTailnetPeerStatusByCoordinator", s.Subtest(func(db database.Store, check *expects) {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
check.Args(database.UpdateTailnetPeerStatusByCoordinatorParams{Status: database.TailnetStatusOk}).
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTailnetCoordinator, policy.ActionUpdate)
}))
}
@@ -4787,21 +4757,18 @@ func (s *MethodTestSuite) TestNotifications() {
dbtestutil.DisableForeignKeysAndTriggers(s.T(), db)
user := dbgen.User(s.T(), db, database.User{})
check.Args(user.ID).Asserts(rbac.ResourceNotificationTemplate, policy.ActionRead).
ErrorsWithPG(sql.ErrNoRows).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
ErrorsWithPG(sql.ErrNoRows)
}))
s.Run("GetNotificationTemplatesByKind", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.NotificationTemplateKindSystem).
Asserts().
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts()
// TODO(dannyk): add support for other database.NotificationTemplateKind types once implemented.
}))
s.Run("UpdateNotificationTemplateMethodByID", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.UpdateNotificationTemplateMethodByIDParams{
Method: database.NullNotificationMethod{NotificationMethod: database.NotificationMethodWebhook, Valid: true},
ID: notifications.TemplateWorkspaceDormant,
}).Asserts(rbac.ResourceNotificationTemplate, policy.ActionUpdate).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
}).Asserts(rbac.ResourceNotificationTemplate, policy.ActionUpdate)
}))
// Notification preferences
@@ -5115,8 +5082,7 @@ func (s *MethodTestSuite) TestPrebuilds() {
rbac.ResourceWorkspace.WithOwner(user.ID.String()).InOrg(org.ID), policy.ActionCreate,
template, policy.ActionRead,
template, policy.ActionUse,
).ErrorsWithInMemDB(dbmem.ErrUnimplemented).
ErrorsWithPG(sql.ErrNoRows)
).Errors(sql.ErrNoRows)
}))
s.Run("GetPrebuildMetrics", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
@@ -5130,29 +5096,24 @@ func (s *MethodTestSuite) TestPrebuilds() {
}))
s.Run("CountInProgressPrebuilds", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead)
}))
s.Run("GetPresetsAtFailureLimit", s.Subtest(func(_ database.Store, check *expects) {
check.Args(int64(0)).
Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights)
}))
s.Run("GetPresetsBackoff", s.Subtest(func(_ database.Store, check *expects) {
check.Args(time.Time{}).
Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTemplate.All(), policy.ActionViewInsights)
}))
s.Run("GetRunningPrebuiltWorkspaces", s.Subtest(func(_ database.Store, check *expects) {
check.Args().
Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceWorkspace.All(), policy.ActionRead)
}))
s.Run("GetTemplatePresetsWithPrebuilds", s.Subtest(func(db database.Store, check *expects) {
user := dbgen.User(s.T(), db, database.User{})
check.Args(uuid.NullUUID{UUID: user.ID, Valid: true}).
Asserts(rbac.ResourceTemplate.All(), policy.ActionRead).
ErrorsWithInMemDB(dbmem.ErrUnimplemented)
Asserts(rbac.ResourceTemplate.All(), policy.ActionRead)
}))
s.Run("GetPresetByID", s.Subtest(func(db database.Store, check *expects) {
org := dbgen.Organization(s.T(), db, database.Organization{})
coderd/database/dbmem/dbmem.go
-14242
View File
File diff suppressed because it is too large Load Diff
coderd/database/dbmem/dbmem_test.go
-209
View File
@@ -1,209 +0,0 @@
package dbmem_test
import (
"context"
"database/sql"
"sort"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/dbtime"
)
// test that transactions don't deadlock, and that we don't see intermediate state.
func TestInTx(t *testing.T) {
t.Parallel()
uut := dbmem.New()
inTx := make(chan any)
queriesDone := make(chan any)
queriesStarted := make(chan any)
go func() {
err := uut.InTx(func(tx database.Store) error {
close(inTx)
_, err := tx.InsertOrganization(context.Background(), database.InsertOrganizationParams{
Name: "1",
})
assert.NoError(t, err)
<-queriesStarted
time.Sleep(5 * time.Millisecond)
_, err = tx.InsertOrganization(context.Background(), database.InsertOrganizationParams{
Name: "2",
})
assert.NoError(t, err)
return nil
}, nil)
assert.NoError(t, err)
}()
var nums []int
go func() {
<-inTx
for i := 0; i < 20; i++ {
orgs, err := uut.GetOrganizations(context.Background(), database.GetOrganizationsParams{})
if err != nil {
assert.ErrorIs(t, err, sql.ErrNoRows)
}
nums = append(nums, len(orgs))
time.Sleep(time.Millisecond)
}
close(queriesDone)
}()
close(queriesStarted)
<-queriesDone
// ensure we never saw 1 org, only 0 or 2.
for i := 0; i < 20; i++ {
assert.NotEqual(t, 1, nums[i])
}
}
// TestUserOrder ensures that the fake database returns users sorted by username.
func TestUserOrder(t *testing.T) {
t.Parallel()
db := dbmem.New()
now := dbtime.Now()
usernames := []string{"b-user", "d-user", "a-user", "c-user", "e-user"}
for _, username := range usernames {
dbgen.User(t, db, database.User{Username: username, CreatedAt: now})
}
users, err := db.GetUsers(context.Background(), database.GetUsersParams{})
require.NoError(t, err)
require.Lenf(t, users, len(usernames), "expected %d users", len(usernames))
sort.Strings(usernames)
for i, user := range users {
require.Equal(t, usernames[i], user.Username)
}
}
func TestProxyByHostname(t *testing.T) {
t.Parallel()
db := dbmem.New()
// Insert a bunch of different proxies.
proxies := []struct {
name string
accessURL string
wildcardHostname string
}{
{
name: "one",
accessURL: "https://one.coder.com",
wildcardHostname: "*.wildcard.one.coder.com",
},
{
name: "two",
accessURL: "https://two.coder.com",
wildcardHostname: "*--suffix.two.coder.com",
},
}
for _, p := range proxies {
dbgen.WorkspaceProxy(t, db, database.WorkspaceProxy{
Name: p.name,
Url: p.accessURL,
WildcardHostname: p.wildcardHostname,
})
}
cases := []struct {
name string
testHostname string
allowAccessURL bool
allowWildcardHost bool
matchProxyName string
}{
{
name: "NoMatch",
testHostname: "test.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "",
},
{
name: "MatchAccessURL",
testHostname: "one.coder.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "one",
},
{
name: "MatchWildcard",
testHostname: "something.wildcard.one.coder.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "one",
},
{
name: "MatchSuffix",
testHostname: "something--suffix.two.coder.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "two",
},
{
name: "ValidateHostname/1",
testHostname: ".*ne.coder.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "",
},
{
name: "ValidateHostname/2",
testHostname: "https://one.coder.com",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "",
},
{
name: "ValidateHostname/3",
testHostname: "one.coder.com:8080/hello",
allowAccessURL: true,
allowWildcardHost: true,
matchProxyName: "",
},
{
name: "IgnoreAccessURLMatch",
testHostname: "one.coder.com",
allowAccessURL: false,
allowWildcardHost: true,
matchProxyName: "",
},
{
name: "IgnoreWildcardMatch",
testHostname: "hi.wildcard.one.coder.com",
allowAccessURL: true,
allowWildcardHost: false,
matchProxyName: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
t.Parallel()
proxy, err := db.GetWorkspaceProxyByHostname(context.Background(), database.GetWorkspaceProxyByHostnameParams{
Hostname: c.testHostname,
AllowAccessUrl: c.allowAccessURL,
AllowWildcardHostname: c.allowWildcardHost,
})
if c.matchProxyName == "" {
require.ErrorIs(t, err, sql.ErrNoRows)
require.Empty(t, proxy)
} else {
require.NoError(t, err)
require.NotEmpty(t, proxy)
require.Equal(t, c.matchProxyName, proxy.Name)
}
})
}
}
coderd/database/dbtestutil/db.go
+43 -46
View File
@@ -20,14 +20,15 @@ import (
"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/testutil"
)
// WillUsePostgres returns true if a call to NewDB() will return a real, postgres-backed Store and Pubsub.
// TODO(hugodutka): since we removed the in-memory database, this is always true,
// and we need to remove this function. https://github.com/coder/internal/issues/758
func WillUsePostgres() bool {
return os.Getenv("DB") != ""
return true
}
type options struct {
@@ -109,52 +110,48 @@ func NewDB(t testing.TB, opts ...Option) (database.Store, pubsub.Pubsub) {
var db database.Store
var ps pubsub.Pubsub
if WillUsePostgres() {
connectionURL := os.Getenv("CODER_PG_CONNECTION_URL")
if connectionURL == "" && o.url != "" {
connectionURL = o.url
}
if connectionURL == "" {
var err error
connectionURL, err = Open(t)
require.NoError(t, err)
}
if o.fixedTimezone == "" {
// To make sure we find timezone-related issues, we set the timezone
// of the database to a non-UTC one.
// The below was picked due to the following properties:
// - It has a non-UTC offset
// - It has a fractional hour UTC offset
// - It includes a daylight savings time component
o.fixedTimezone = DefaultTimezone
}
dbName := dbNameFromConnectionURL(t, connectionURL)
setDBTimezone(t, connectionURL, dbName, o.fixedTimezone)
sqlDB, err := sql.Open("postgres", connectionURL)
require.NoError(t, err)
t.Cleanup(func() {
_ = sqlDB.Close()
})
if o.returnSQLDB != nil {
o.returnSQLDB(sqlDB)
}
if o.dumpOnFailure {
t.Cleanup(func() { DumpOnFailure(t, connectionURL) })
}
// Unit tests should not retry serial transaction failures.
db = database.New(sqlDB, database.WithSerialRetryCount(1))
ps, err = pubsub.New(context.Background(), o.logger, sqlDB, connectionURL)
require.NoError(t, err)
t.Cleanup(func() {
_ = ps.Close()
})
} else {
db = dbmem.New()
ps = pubsub.NewInMemory()
connectionURL := os.Getenv("CODER_PG_CONNECTION_URL")
if connectionURL == "" && o.url != "" {
connectionURL = o.url
}
if connectionURL == "" {
var err error
connectionURL, err = Open(t)
require.NoError(t, err)
}
if o.fixedTimezone == "" {
// To make sure we find timezone-related issues, we set the timezone
// of the database to a non-UTC one.
// The below was picked due to the following properties:
// - It has a non-UTC offset
// - It has a fractional hour UTC offset
// - It includes a daylight savings time component
o.fixedTimezone = DefaultTimezone
}
dbName := dbNameFromConnectionURL(t, connectionURL)
setDBTimezone(t, connectionURL, dbName, o.fixedTimezone)
sqlDB, err := sql.Open("postgres", connectionURL)
require.NoError(t, err)
t.Cleanup(func() {
_ = sqlDB.Close()
})
if o.returnSQLDB != nil {
o.returnSQLDB(sqlDB)
}
if o.dumpOnFailure {
t.Cleanup(func() { DumpOnFailure(t, connectionURL) })
}
// Unit tests should not retry serial transaction failures.
db = database.New(sqlDB, database.WithSerialRetryCount(1))
ps, err = pubsub.New(context.Background(), o.logger, sqlDB, connectionURL)
require.NoError(t, err)
t.Cleanup(func() {
_ = ps.Close()
})
return db, ps
}
coderd/database/oidcclaims_test.go
-1
View File
@@ -222,7 +222,6 @@ func (g userGenerator) withLink(lt database.LoginType, rawJSON json.RawMessage)
err := sql.UpdateUserLinkRawJSON(context.Background(), user.ID, rawJSON)
require.NoError(t, err)
} else {
// no need to test the json key logic in dbmem. Everything is type safe.
var claims database.UserLinkClaims
err := json.Unmarshal(rawJSON, &claims)
require.NoError(t, err)
coderd/database/querier_test.go
-1
View File
@@ -1400,7 +1400,6 @@ func TestGetUsers_IncludeSystem(t *testing.T) {
// Given: a system user
// postgres: introduced by migration coderd/database/migrations/00030*_system_user.up.sql
// dbmem: created in dbmem/dbmem.go
db, _ := dbtestutil.NewDB(t)
other := dbgen.User(t, db, database.User{})
users, err := db.GetUsers(ctx, database.GetUsersParams{
coderd/notifications/notifications_test.go
+1 -1
View File
@@ -769,7 +769,7 @@ func TestNotificationTemplates_Golden(t *testing.T) {
hello = "localhost"
from = "system@coder.com"
hint = "run \"DB=ci make gen/golden-files\" and commit the changes"
hint = "run \"make gen/golden-files\" and commit the changes"
)
tests := []struct {
coderd/users_test.go
-10
View File
@@ -32,7 +32,6 @@ import (
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbfake"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/util/ptr"
@@ -1794,15 +1793,6 @@ func TestUsersFilter(t *testing.T) {
}
}
// TODO: This can be removed with dbmem
if !dbtestutil.WillUsePostgres() {
for i := range matched.Users {
if len(matched.Users[i].OrganizationIDs) == 0 {
matched.Users[i].OrganizationIDs = nil
}
}
}
require.ElementsMatch(t, exp, matched.Users, "expected users returned")
})
}
codersdk/deployment.go
-10
View File
@@ -354,7 +354,6 @@ type DeploymentValues struct {
ProxyTrustedHeaders serpent.StringArray `json:"proxy_trusted_headers,omitempty" typescript:",notnull"`
ProxyTrustedOrigins serpent.StringArray `json:"proxy_trusted_origins,omitempty" typescript:",notnull"`
CacheDir serpent.String `json:"cache_directory,omitempty" typescript:",notnull"`
InMemoryDatabase serpent.Bool `json:"in_memory_database,omitempty" typescript:",notnull"`
EphemeralDeployment serpent.Bool `json:"ephemeral_deployment,omitempty" typescript:",notnull"`
PostgresURL serpent.String `json:"pg_connection_url,omitempty" typescript:",notnull"`
PostgresAuth string `json:"pg_auth,omitempty" typescript:",notnull"`
@@ -2404,15 +2403,6 @@ func (c *DeploymentValues) Options() serpent.OptionSet {
Value: &c.CacheDir,
YAML: "cacheDir",
},
{
Name: "In Memory Database",
Description: "Controls whether data will be stored in an in-memory database.",
Flag: "in-memory",
Env: "CODER_IN_MEMORY",
Hidden: true,
Value: &c.InMemoryDatabase,
YAML: "inMemoryDatabase",
},
{
Name: "Ephemeral Deployment",
Description: "Controls whether Coder data, including built-in Postgres, will be stored in a temporary directory and deleted when the server is stopped.",
docs/about/contributing/backend.md
-1
View File
@@ -68,7 +68,6 @@ The Coder backend is organized into multiple packages and directories, each with
* [dbauthz](https://github.com/coder/coder/tree/main/coderd/database/dbauthz): AuthZ wrappers for database queries, ideally, every query should verify first if the accessor is eligible to see the query results.
* [dbfake](https://github.com/coder/coder/tree/main/coderd/database/dbfake): helper functions to quickly prepare the initial database state for testing purposes (e.g. create N healthy workspaces and templates), operates on higher level than [dbgen](https://github.com/coder/coder/tree/main/coderd/database/dbgen)
* [dbgen](https://github.com/coder/coder/tree/main/coderd/database/dbgen): helper functions to insert raw records to the database store, used for testing purposes
* [dbmem](https://github.com/coder/coder/tree/main/coderd/database/dbmem): in-memory implementation of the database store, ideally, every real query should have a complimentary Go implementation
* [dbmock](https://github.com/coder/coder/tree/main/coderd/database/dbmock): a store wrapper for database queries, useful to verify if the function has been called, used for testing purposes
* [dbpurge](https://github.com/coder/coder/tree/main/coderd/database/dbpurge): simple wrapper for periodic database cleanup operations
* [migrations](https://github.com/coder/coder/tree/main/coderd/database/migrations): an ordered list of up/down database migrations, use `./create_migration.sh my_migration_name` to modify the database schema
docs/reference/api/general.md
Generated
-1
View File
@@ -265,7 +265,6 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
"same_site": "string",
"secure_auth_cookie": true
},
"in_memory_database": true,
"job_hang_detector_interval": 0,
"logging": {
"human": "string",
docs/reference/api/schemas.md
Generated
-3
View File
@@ -1987,7 +1987,6 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
"same_site": "string",
"secure_auth_cookie": true
},
"in_memory_database": true,
"job_hang_detector_interval": 0,
"logging": {
"human": "string",
@@ -2475,7 +2474,6 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
"same_site": "string",
"secure_auth_cookie": true
},
"in_memory_database": true,
"job_hang_detector_interval": 0,
"logging": {
"human": "string",
@@ -2772,7 +2770,6 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
| `hide_ai_tasks` | boolean | false | | |
| `http_address` | string | false | | Http address is a string because it may be set to zero to disable. |
| `http_cookies` | [codersdk.HTTPCookieConfig](#codersdkhttpcookieconfig) | false | | |
| `in_memory_database` | boolean | false | | |
| `job_hang_detector_interval` | integer | false | | |
| `logging` | [codersdk.LoggingConfig](#codersdkloggingconfig) | false | | |
| `metrics_cache_refresh_interval` | integer | false | | |
enterprise/cli/server_test.go
-3
View File
@@ -18,9 +18,6 @@ import (
)
func dbArg(t *testing.T) string {
if !dbtestutil.WillUsePostgres() {
return "--in-memory"
}
dbURL, err := dbtestutil.Open(t)
require.NoError(t, err)
return "--postgres-url=" + dbURL
enterprise/coderd/coderd.go
+1 -7
View File
@@ -780,13 +780,7 @@ func (api *API) updateEntitlements(ctx context.Context) error {
if initial, changed, enabled := featureChanged(codersdk.FeatureHighAvailability); shouldUpdate(initial, changed, enabled) {
var coordinator agpltailnet.Coordinator
// If HA is enabled, but the database is in-memory, we can't actually
// run HA and the PG coordinator. So throw a log line, and continue to use
// the in memory AGPL coordinator.
if enabled && api.DeploymentValues.InMemoryDatabase.Value() {
api.Logger.Warn(ctx, "high availability is enabled, but cannot be configured due to the database being set to in-memory")
}
if enabled && !api.DeploymentValues.InMemoryDatabase.Value() {
if enabled {
haCoordinator, err := tailnet.NewPGCoord(api.ctx, api.Logger, api.Pubsub, api.Database)
if err != nil {
api.Logger.Error(ctx, "unable to set up high availability coordinator", slog.Error(err))
enterprise/coderd/coderdenttest/coderdenttest.go
-3
View File
@@ -22,7 +22,6 @@ import (
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmem"
"github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/drpcsdk"
@@ -149,8 +148,6 @@ func NewWithAPI(t *testing.T, options *Options) (
// we check for the in-memory test types so that the real types don't have to exported
_, ok := coderAPI.Pubsub.(*pubsub.MemoryPubsub)
require.False(t, ok, "FeatureHighAvailability is incompatible with MemoryPubsub")
_, ok = coderAPI.Database.(*dbmem.FakeQuerier)
require.False(t, ok, "FeatureHighAvailability is incompatible with dbmem")
}
}
_ = AddLicense(t, client, lo)
scripts/dbgen/main.go
+2 -81
View File
@@ -7,7 +7,6 @@ import (
"go/format"
"go/token"
"os"
"path"
"path/filepath"
"reflect"
"runtime"
@@ -52,14 +51,6 @@ func run() error {
return err
}
databasePath := filepath.Join(localPath, "..", "..", "..", "coderd", "database")
err = orderAndStubDatabaseFunctions(filepath.Join(databasePath, "dbmem", "dbmem.go"), "q", "FakeQuerier", func(_ stubParams) string {
return `panic("not implemented")`
})
if err != nil {
return xerrors.Errorf("stub dbmem: %w", err)
}
err = orderAndStubDatabaseFunctions(filepath.Join(databasePath, "dbmetrics", "querymetrics.go"), "m", "queryMetricsStore", func(params stubParams) string {
return fmt.Sprintf(`
start := time.Now()
@@ -257,13 +248,13 @@ func orderAndStubDatabaseFunctions(filePath, receiver, structName string, stub f
contents, err := os.ReadFile(filePath)
if err != nil {
return xerrors.Errorf("read dbmem: %w", err)
return xerrors.Errorf("read file: %w", err)
}
// Required to preserve imports!
f, err := decorator.NewDecoratorWithImports(token.NewFileSet(), packageName, goast.New()).Parse(contents)
if err != nil {
return xerrors.Errorf("parse dbmem: %w", err)
return xerrors.Errorf("parse file: %w", err)
}
pointer := false
@@ -298,76 +289,6 @@ func orderAndStubDatabaseFunctions(filePath, receiver, structName string, stub f
for _, fn := range funcs {
var bodyStmts []dst.Stmt
// Add input validation, only relevant for dbmem.
if strings.Contains(filePath, "dbmem") && len(fn.Func.Params.List) == 2 && fn.Func.Params.List[1].Names[0].Name == "arg" {
/*
err := validateDatabaseType(arg)
if err != nil {
return database.User{}, err
}
*/
bodyStmts = append(bodyStmts, &dst.AssignStmt{
Lhs: []dst.Expr{dst.NewIdent("err")},
Tok: token.DEFINE,
Rhs: []dst.Expr{
&dst.CallExpr{
Fun: &dst.Ident{
Name: "validateDatabaseType",
},
Args: []dst.Expr{dst.NewIdent("arg")},
},
},
})
returnStmt := &dst.ReturnStmt{
Results: []dst.Expr{}, // Filled below.
}
bodyStmts = append(bodyStmts, &dst.IfStmt{
Cond: &dst.BinaryExpr{
X: dst.NewIdent("err"),
Op: token.NEQ,
Y: dst.NewIdent("nil"),
},
Body: &dst.BlockStmt{
List: []dst.Stmt{
returnStmt,
},
},
Decs: dst.IfStmtDecorations{
NodeDecs: dst.NodeDecs{
After: dst.EmptyLine,
},
},
})
for _, r := range fn.Func.Results.List {
switch typ := r.Type.(type) {
case *dst.StarExpr, *dst.ArrayType, *dst.SelectorExpr:
returnStmt.Results = append(returnStmt.Results, dst.NewIdent("nil"))
case *dst.Ident:
if typ.Path != "" {
returnStmt.Results = append(returnStmt.Results, dst.NewIdent(fmt.Sprintf("%s.%s{}", path.Base(typ.Path), typ.Name)))
} else {
switch typ.Name {
case "uint8", "uint16", "uint32", "uint64", "uint", "uintptr",
"int8", "int16", "int32", "int64", "int",
"byte", "rune",
"float32", "float64",
"complex64", "complex128":
returnStmt.Results = append(returnStmt.Results, dst.NewIdent("0"))
case "string":
returnStmt.Results = append(returnStmt.Results, dst.NewIdent("\"\""))
case "bool":
returnStmt.Results = append(returnStmt.Results, dst.NewIdent("false"))
case "error":
returnStmt.Results = append(returnStmt.Results, dst.NewIdent("err"))
default:
panic(fmt.Sprintf("unknown ident: %#v", r.Type))
}
}
default:
panic(fmt.Sprintf("unknown return type: %T", r.Type))
}
}
}
decl, ok := declByName[fn.Name]
if !ok {
typeName := structName
site/e2e/playwright.config.ts
+1 -1
View File
@@ -72,7 +72,7 @@ export default defineConfig({
"--global-config $(mktemp -d -t e2e-XXXXXXXXXX)",
`--access-url=http://localhost:${coderPort}`,
`--http-address=0.0.0.0:${coderPort}`,
"--in-memory",
"--ephemeral",
"--telemetry=false",
"--dangerous-disable-rate-limits",
"--provisioner-daemons 10",
site/src/api/typesGenerated.ts
Generated
-1
View File
@@ -668,7 +668,6 @@ export interface DeploymentValues {
readonly proxy_trusted_headers?: string;
readonly proxy_trusted_origins?: string;
readonly cache_directory?: string;
readonly in_memory_database?: boolean;
readonly ephemeral_deployment?: boolean;
readonly pg_connection_url?: string;
readonly pg_auth?: string;