From 628750232fc23deb3a853be8e24742ca5cb2139d Mon Sep 17 00:00:00 2001 From: Ethan <39577870+ethanndickson@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:30:04 +1000 Subject: [PATCH] fix: delete workspace agent stats after 180 days (#14489) Fixes #13430. The test for purging old workspace agent stats from the DB was consistently failing when ran with Postgres towards the end of the month, but not with the in-memory DB. This was because month intervals are calculated differently for `time.Time` and the `interval` type in Postgres: ``` ethan=# SELECT '2024-08-30'::DATE AS original_date, ('2024-08-30'::DATE - INTERVAL '6 months') AS sub_date; original_date | sub_date ---------------+--------------------- 2024-08-30 | 2024-02-29 00:00:00 (1 row) ``` Using `func (t Time) AddDate(years int, months int, days int) Time`, where `months` is `-6`: ``` Original: 2024-08-30 00:00:00 +0000 UTC 6 Months Earlier: 2024-03-01 00:00:00 +0000 UTC ``` Since 6 months was chosen arbitrarily, we should be able to change it to 180 days, to remove any ambiguity between the in-memory DB, and the Postgres DB. The alternative solution would involve implementing Postgres' month interval algorithm in Go. The UI only shows stats as old as 168 days (24 weeks), so a frontend change isn't required for the extra days of stats we lose in some cases. --- coderd/database/dbmem/dbmem.go | 6 +++--- coderd/database/dbpurge/dbpurge_test.go | 20 +++++++++---------- coderd/database/queries.sql.go | 4 ++-- .../database/queries/workspaceagentstats.sql | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index df0b414581..826c64cc30 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1749,10 +1749,10 @@ func (q *FakeQuerier) DeleteOldWorkspaceAgentStats(_ context.Context) error { -- use between 15 mins and 1 hour of data. We keep a -- little bit more (1 day) just in case. MAX(start_time) - '1 days'::interval, - -- Fall back to 6 months ago if there are no template + -- Fall back to ~6 months ago if there are no template -- usage stats so that we don't delete the data before -- it's rolled up. - NOW() - '6 months'::interval + NOW() - '180 days'::interval ) FROM template_usage_stats @@ -1778,7 +1778,7 @@ func (q *FakeQuerier) DeleteOldWorkspaceAgentStats(_ context.Context) error { } // COALESCE if limit.IsZero() { - limit = now.AddDate(0, -6, 0) + limit = now.AddDate(0, 0, -180) } var validStats []database.WorkspaceAgentStat diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index a79bb1b6c1..04c2aaf124 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -86,27 +86,27 @@ func TestDeleteOldWorkspaceAgentStats(t *testing.T) { // conflicts, verifying DST behavior is beyond the scope of this // test. // Let's use RxBytes to identify stat entries. - // Stat inserted 6 months + 2 hour ago, should be deleted. + // Stat inserted 180 days + 2 hour ago, should be deleted. first := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ - CreatedAt: now.AddDate(0, -6, 0).Add(-2 * time.Hour), + CreatedAt: now.AddDate(0, 0, -180).Add(-2 * time.Hour), ConnectionCount: 1, ConnectionMedianLatencyMS: 1, RxBytes: 1111, SessionCountSSH: 1, }) - // Stat inserted 6 months - 2 hour ago, should not be deleted before rollup. + // Stat inserted 180 days - 2 hour ago, should not be deleted before rollup. second := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ - CreatedAt: now.AddDate(0, -6, 0).Add(2 * time.Hour), + CreatedAt: now.AddDate(0, 0, -180).Add(2 * time.Hour), ConnectionCount: 1, ConnectionMedianLatencyMS: 1, RxBytes: 2222, SessionCountSSH: 1, }) - // Stat inserted 6 months - 1 day - 4 hour ago, should not be deleted at all. + // Stat inserted 179 days - 4 hour ago, should not be deleted at all. third := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ - CreatedAt: now.AddDate(0, -6, 0).AddDate(0, 0, 1).Add(4 * time.Hour), + CreatedAt: now.AddDate(0, 0, -179).Add(4 * time.Hour), ConnectionCount: 1, ConnectionMedianLatencyMS: 1, RxBytes: 3333, @@ -121,8 +121,8 @@ func TestDeleteOldWorkspaceAgentStats(t *testing.T) { var stats []database.GetWorkspaceAgentStatsRow var err error require.Eventuallyf(t, func() bool { - // Query all stats created not earlier than 7 months ago - stats, err = db.GetWorkspaceAgentStats(ctx, now.AddDate(0, -7, 0)) + // Query all stats created not earlier than ~7 months ago + stats, err = db.GetWorkspaceAgentStats(ctx, now.AddDate(0, 0, -210)) if err != nil { return false } @@ -144,8 +144,8 @@ func TestDeleteOldWorkspaceAgentStats(t *testing.T) { // then require.Eventuallyf(t, func() bool { - // Query all stats created not earlier than 7 months ago - stats, err = db.GetWorkspaceAgentStats(ctx, now.AddDate(0, -7, 0)) + // Query all stats created not earlier than ~7 months ago + stats, err = db.GetWorkspaceAgentStats(ctx, now.AddDate(0, 0, -210)) if err != nil { return false } diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 67a8a56274..8e8036d621 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -11481,10 +11481,10 @@ WHERE -- use between 15 mins and 1 hour of data. We keep a -- little bit more (1 day) just in case. MAX(start_time) - '1 days'::interval, - -- Fall back to 6 months ago if there are no template + -- Fall back to ~6 months ago if there are no template -- usage stats so that we don't delete the data before -- it's rolled up. - NOW() - '6 months'::interval + NOW() - '180 days'::interval ) FROM template_usage_stats diff --git a/coderd/database/queries/workspaceagentstats.sql b/coderd/database/queries/workspaceagentstats.sql index f96140c87c..ccc6d7ae21 100644 --- a/coderd/database/queries/workspaceagentstats.sql +++ b/coderd/database/queries/workspaceagentstats.sql @@ -78,10 +78,10 @@ WHERE -- use between 15 mins and 1 hour of data. We keep a -- little bit more (1 day) just in case. MAX(start_time) - '1 days'::interval, - -- Fall back to 6 months ago if there are no template + -- Fall back to ~6 months ago if there are no template -- usage stats so that we don't delete the data before -- it's rolled up. - NOW() - '6 months'::interval + NOW() - '180 days'::interval ) FROM template_usage_stats