feat: add standard encodings to binary cache (#21921)

fixes: https://github.com/coder/internal/issues/1300

Adds brotli and zstd compression to the binary cache. Also refactors coderd's streaming encoding middleware to use the same standard set of compression algorithms, so we have them in one place.

coderd/coderd.go

@@ -21,11 +21,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/andybalholm/brotli"
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/google/uuid"
-	"github.com/klauspost/compress/zstd"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -1972,16 +1970,13 @@ func compressHandler(h http.Handler) http.Handler {
 		"application/*",
 		"image/*",
 	)
-	cmp.SetEncoder("br", func(w io.Writer, level int) io.Writer {
-		return brotli.NewWriterLevel(w, level)
-	})
-	cmp.SetEncoder("zstd", func(w io.Writer, level int) io.Writer {
-		zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level)))
-		if err != nil {
-			panic("invalid zstd compressor: " + err.Error())
-		}
-		return zw
-	})
+	for encoding := range site.StandardEncoders {
+		writeCloserFn := site.StandardEncoders[encoding]
+		cmp.SetEncoder(encoding, func(w io.Writer, level int) io.Writer {
+			writeCloser := writeCloserFn(w, level)
+			return writeCloser
+		})
+	}
 
 	return cmp.Handler(h)
 }

site/bin.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/andybalholm/brotli"
 	"github.com/klauspost/compress/zstd"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/singleflight"
@@ -35,6 +36,19 @@ type binHandler struct {
 	handler       http.Handler
 }
 
+var StandardEncoders = map[string]func(w io.Writer, level int) io.WriteCloser{
+	"br": func(w io.Writer, level int) io.WriteCloser {
+		return brotli.NewWriterLevel(w, level)
+	},
+	"zstd": func(w io.Writer, level int) io.WriteCloser {
+		zw, err := zstd.NewWriter(w, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level)))
+		if err != nil {
+			panic("invalid zstd compressor: " + err.Error())
+		}
+		return zw
+	},
+}
+
 func (h *binHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	if !strings.HasPrefix(r.URL.Path, "/bin/") {
 		rw.WriteHeader(http.StatusNotFound)
@@ -120,7 +134,11 @@ func newBinHandler(options *Options) (*binHandler, error) {
 		metadataCache: newBinMetadataCache(binFS, binHashes),
 	}
 	if compressedCacheDir != "" {
-		h.handler = cachecompress.NewCompressor(options.Logger, CompressionLevel, compressedCacheDir, binFS)
+		cmp := cachecompress.NewCompressor(options.Logger, CompressionLevel, compressedCacheDir, binFS)
+		for encoding, fn := range StandardEncoders {
+			cmp.SetEncoder(encoding, fn)
+		}
+		h.handler = cmp
 	} else {
 		h.handler = http.FileServer(binFS)
 	}