diff --git a/app/__test__/_assets/.gitignore b/app/__test__/_assets/.gitignore new file mode 100644 index 00000000..e540f5b3 --- /dev/null +++ b/app/__test__/_assets/.gitignore @@ -0,0 +1 @@ +tmp/* \ No newline at end of file diff --git a/app/__test__/_assets/image.png b/app/__test__/_assets/image.png new file mode 100644 index 00000000..2ebf8fc8 Binary files /dev/null and b/app/__test__/_assets/image.png differ diff --git a/app/__test__/api/MediaApi.spec.ts b/app/__test__/api/MediaApi.spec.ts new file mode 100644 index 00000000..73877924 --- /dev/null +++ b/app/__test__/api/MediaApi.spec.ts @@ -0,0 +1,143 @@ +/// +import { describe, expect, it } from "bun:test"; +import { Hono } from "hono"; +import { getFileFromContext, isFile, isReadableStream } from "../../src/core/utils"; +import { MediaApi } from "../../src/media/api/MediaApi"; +import { assetsPath, assetsTmpPath } from "../helper"; + +const mockedBackend = new Hono() + .basePath("/api/media") + .post("/upload/:name", async (c) => { + const { name } = c.req.param(); + const body = await getFileFromContext(c); + return c.json({ name, is_file: isFile(body), size: body.size }); + }) + .get("/file/:name", async (c) => { + const { name } = c.req.param(); + const file = Bun.file(`${assetsPath}/${name}`); + return new Response(file, { + headers: { + "Content-Type": file.type, + "Content-Length": file.size.toString() + } + }); + }); + +describe("MediaApi", () => { + it("should give correct file upload url", () => { + const host = "http://localhost"; + const basepath = "/api/media"; + // @ts-ignore tests + const api = new MediaApi({ + host, + basepath + }); + expect(api.getFileUploadUrl({ path: "path" })).toBe(`${host}${basepath}/upload/path`); + }); + + it("should have correct upload headers", () => { + // @ts-ignore tests + const api = new MediaApi({ + token: "token" + }); + expect(api.getUploadHeaders().get("Authorization")).toBe("Bearer token"); + }); + + it("should get file: native", async () => { + const name = "image.png"; + const path = `${assetsTmpPath}/${name}`; + const res = await mockedBackend.request("/api/media/file/" + name); + await Bun.write(path, res); + + const file = await Bun.file(path); + expect(file.size).toBeGreaterThan(0); + expect(file.type).toBe("image/png"); + await file.delete(); + }); + + it("download", async () => { + // @ts-ignore tests + const api = new MediaApi({}, mockedBackend.request); + + const name = "image.png"; + const file = await api.download(name); + expect(isFile(file)).toBe(true); + expect(file.size).toBeGreaterThan(0); + expect(file.type).toBe("image/png"); + expect(file.name).toContain(name); + }); + + it("getFile", async () => { + // @ts-ignore tests + const api = new MediaApi({}, mockedBackend.request); + + const name = "image.png"; + const res = await api.getFile(name); + expect(res.ok).toBe(true); + // make sure it's a normal api request as usual + expect(res.res.ok).toBe(true); + expect(isReadableStream(res)).toBe(true); + expect(isReadableStream(res.body)).toBe(true); + expect(isReadableStream(res.res.body)).toBe(true); + + const blob = await res.res.blob(); + expect(isFile(blob)).toBe(true); + expect(blob.size).toBeGreaterThan(0); + expect(blob.type).toBe("image/png"); + expect(blob.name).toContain(name); + }); + + it("getFileStream", async () => { + // @ts-ignore tests + const api = new MediaApi({}, mockedBackend.request); + + const name = "image.png"; + const res = await api.getFileStream(name); + expect(isReadableStream(res)).toBe(true); + + const blob = await new Response(res).blob(); + expect(isFile(blob)).toBe(true); + expect(blob.size).toBeGreaterThan(0); + expect(blob.type).toBe("image/png"); + expect(blob.name).toContain(name); + }); + + it("should upload file in various ways", async () => { + // @ts-ignore tests + const api = new MediaApi({}, mockedBackend.request); + const file = Bun.file(`${assetsPath}/image.png`); + + async function matches(req: Promise, filename: string) { + const res: any = await req; + expect(res.name).toBe(filename); + expect(res.is_file).toBe(true); + expect(res.size).toBe(file.size); + } + + const url = "http://localhost/api/media/file/image.png"; + + // upload bun file + await matches(api.upload(file as any, "bunfile.png"), "bunfile.png"); + + // upload via request + await matches(api.upload(new Request(url), "request.png"), "request.png"); + + // upload via url + await matches(api.upload(url, "url.png"), "url.png"); + + // upload via response + { + const response = await mockedBackend.request(url); + await matches(api.upload(response, "response.png"), "response.png"); + } + + // upload via readable from bun + await matches(await api.upload(file.stream(), "readable.png"), "readable.png"); + + // upload via readable from response + { + const response = (await mockedBackend.request(url)) as Response; + await matches(await api.upload(response.body!, "readable.png"), "readable.png"); + } + }); +}); diff --git a/app/__test__/core/utils.spec.ts b/app/__test__/core/utils.spec.ts index b484be84..3de8a48f 100644 --- a/app/__test__/core/utils.spec.ts +++ b/app/__test__/core/utils.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { Perf } from "../../src/core/utils"; +import { Perf, isBlob, ucFirst } from "../../src/core/utils"; import * as utils from "../../src/core/utils"; async function wait(ms: number) { @@ -75,6 +75,57 @@ describe("Core Utils", async () => { const result3 = utils.encodeSearch(obj3, { encode: true }); expect(result3).toBe("id=123&name=%7B%22test%22%3A%22test%22%7D"); }); + + describe("guards", () => { + const types = { + blob: new Blob(), + file: new File([""], "file.txt"), + stream: new ReadableStream(), + arrayBuffer: new ArrayBuffer(10), + arrayBufferView: new Uint8Array(new ArrayBuffer(10)) + }; + + const fns = [ + [utils.isReadableStream, "stream"], + [utils.isBlob, "blob", ["stream", "arrayBuffer", "arrayBufferView"]], + [utils.isFile, "file", ["stream", "arrayBuffer", "arrayBufferView"]], + [utils.isArrayBuffer, "arrayBuffer"], + [utils.isArrayBufferView, "arrayBufferView"] + ] as const; + + const additional = [0, 0.0, "", null, undefined, {}, []]; + + for (const [fn, type, _to_test] of fns) { + test(`is${ucFirst(type)}`, () => { + const to_test = _to_test ?? (Object.keys(types) as string[]); + for (const key of to_test) { + const value = types[key as keyof typeof types]; + const result = fn(value); + expect(result).toBe(key === type); + } + + for (const value of additional) { + const result = fn(value); + expect(result).toBe(false); + } + }); + } + }); + + test("getContentName", () => { + const name = "test.json"; + const text = "attachment; filename=" + name; + const headers = new Headers({ + "Content-Disposition": text + }); + const request = new Request("http://example.com", { + headers + }); + + expect(utils.getContentName(text)).toBe(name); + expect(utils.getContentName(headers)).toBe(name); + expect(utils.getContentName(request)).toBe(name); + }); }); describe("perf", async () => { @@ -134,8 +185,8 @@ describe("Core Utils", async () => { [true, true, true], [true, false, false], [false, false, true], - [1, NaN, false], - [NaN, NaN, true], + [1, Number.NaN, false], + [Number.NaN, Number.NaN, true], [null, null, true], [null, undefined, false], [undefined, undefined, true], diff --git a/app/__test__/helper.ts b/app/__test__/helper.ts index f07cd34c..8cafff28 100644 --- a/app/__test__/helper.ts +++ b/app/__test__/helper.ts @@ -68,3 +68,6 @@ export function schemaToEm(s: ReturnType, conn?: Connection): En const connection = conn ? conn : getDummyConnection().dummyConnection; return new EntityManager(Object.values(s.entities), connection, s.relations, s.indices); } + +export const assetsPath = `${import.meta.dir}/_assets`; +export const assetsTmpPath = `${import.meta.dir}/_assets/tmp`; diff --git a/app/__test__/media/MediaController.spec.ts b/app/__test__/media/MediaController.spec.ts index 4816620e..49b09001 100644 --- a/app/__test__/media/MediaController.spec.ts +++ b/app/__test__/media/MediaController.spec.ts @@ -1,56 +1,96 @@ -import { describe, test } from "bun:test"; -import { Hono } from "hono"; -import { Guard } from "../../src/auth"; -import { EventManager } from "../../src/core/events"; -import { EntityManager } from "../../src/data"; -import { AppMedia } from "../../src/media/AppMedia"; -import { MediaController } from "../../src/media/api/MediaController"; -import { getDummyConnection } from "../helper"; +/// -const { dummyConnection, afterAllCleanup } = getDummyConnection(); +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { createApp, registries } from "../../src"; +import { StorageLocalAdapter } from "../../src/adapter/node"; +import { mergeObject, randomString } from "../../src/core/utils"; +import type { TAppMediaConfig } from "../../src/media/media-schema"; +import { assetsPath, assetsTmpPath, disableConsoleLog, enableConsoleLog } from "../helper"; -/** - * R2 - * value: ReadableStream | ArrayBuffer | ArrayBufferView | string | Blob | null, - * Node writefile - * data: string | NodeJS.ArrayBufferView | Iterable | AsyncIterable | Stream, - */ -const ALL_TESTS = !!process.env.ALL_TESTS; -describe.skipIf(ALL_TESTS)("MediaController", () => { - test("..", async () => { - const ctx: any = { - em: new EntityManager([], dummyConnection, []), - guard: new Guard(), - emgr: new EventManager(), - server: new Hono() - }; +beforeAll(() => { + registries.media.register("local", StorageLocalAdapter); +}); - const media = new AppMedia( - // @ts-ignore - { - enabled: true, - adapter: { - type: "s3", - config: { - access_key: process.env.R2_ACCESS_KEY as string, - secret_access_key: process.env.R2_SECRET_ACCESS_KEY as string, - url: process.env.R2_URL as string +const path = `${assetsPath}/image.png`; + +async function makeApp(mediaOverride: Partial = {}) { + const app = createApp({ + initialConfig: { + media: mergeObject( + { + enabled: true, + adapter: { + type: "local", + config: { + path: assetsTmpPath + } } - } - }, - ctx - ); - await media.build(); - const app = new MediaController(media).getController(); + }, + mediaOverride + ) + } + }); - const file = Bun.file(`${import.meta.dir}/adapters/icon.png`); - console.log("file", file); - const form = new FormData(); - form.append("file", file); + await app.build(); + return app; +} - await app.request("/upload/test.png", { +function makeName(ext: string) { + return randomString(10) + "." + ext; +} + +beforeAll(disableConsoleLog); +afterAll(enableConsoleLog); + +describe("MediaController", () => { + test("accepts direct", async () => { + const app = await makeApp(); + + const file = Bun.file(path); + const name = makeName("png"); + const res = await app.server.request("/api/media/upload/" + name, { method: "POST", body: file }); + const result = (await res.json()) as any; + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); + + test("accepts form data", async () => { + const app = await makeApp(); + + const file = Bun.file(path); + const name = makeName("png"); + const form = new FormData(); + form.append("file", file); + + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: form + }); + const result = (await res.json()) as any; + expect(result.name).toBe(name); + + const destFile = Bun.file(assetsTmpPath + "/" + name); + expect(destFile.exists()).resolves.toBe(true); + await destFile.delete(); + }); + + test("limits body", async () => { + const app = await makeApp({ storage: { body_max_size: 1 } }); + + const file = await Bun.file(path); + const name = makeName("png"); + const res = await app.server.request("/api/media/upload/" + name, { + method: "POST", + body: file + }); + + expect(res.status).toBe(413); + expect(await Bun.file(assetsTmpPath + "/" + name).exists()).toBe(false); }); }); diff --git a/app/__test__/media/adapters/StorageLocalAdapter.spec.ts b/app/__test__/media/adapters/StorageLocalAdapter.spec.ts index a7c6d79d..2240aeb2 100644 --- a/app/__test__/media/adapters/StorageLocalAdapter.spec.ts +++ b/app/__test__/media/adapters/StorageLocalAdapter.spec.ts @@ -15,7 +15,7 @@ describe("StorageLocalAdapter", () => { test("puts an object", async () => { objects = (await adapter.listObjects()).length; - expect(await adapter.putObject(filename, file)).toBeString(); + expect(await adapter.putObject(filename, file as unknown as File)).toBeString(); }); test("lists objects", async () => { diff --git a/app/package.json b/app/package.json index c6ceb33f..2ab50e92 100644 --- a/app/package.json +++ b/app/package.json @@ -3,7 +3,7 @@ "type": "module", "sideEffects": false, "bin": "./dist/cli/index.js", - "version": "0.7.1", + "version": "0.8.0-rc.1", "description": "Lightweight Firebase/Supabase alternative built to run anywhere — incl. Next.js, Remix, Astro, Cloudflare, Bun, Node, AWS Lambda & more.", "homepage": "https://bknd.io", "repository": { diff --git a/app/src/core/utils/reqres.ts b/app/src/core/utils/reqres.ts index 21e0e28e..e8fa4d49 100644 --- a/app/src/core/utils/reqres.ts +++ b/app/src/core/utils/reqres.ts @@ -1,3 +1,7 @@ +import { randomString } from "core/utils/strings"; +import type { Context } from "hono"; +import { extension, guess, isMimeType } from "media/storage/mime-types-tiny"; + export function headersToObject(headers: Headers): Record { if (!headers) return {}; return { ...Object.fromEntries(headers.entries()) }; @@ -82,3 +86,259 @@ export function decodeSearch(str) { return out; } + +export function isReadableStream(value: unknown): value is ReadableStream { + return ( + typeof value === "object" && + value !== null && + typeof (value as ReadableStream).getReader === "function" + ); +} + +export function isBlob(value: unknown): value is Blob { + return ( + typeof value === "object" && + value !== null && + typeof (value as Blob).arrayBuffer === "function" && + typeof (value as Blob).type === "string" + ); +} + +export function isFile(value: unknown): value is File { + return ( + isBlob(value) && + typeof (value as File).name === "string" && + typeof (value as File).lastModified === "number" + ); +} + +export function isArrayBuffer(value: unknown): value is ArrayBuffer { + return ( + typeof value === "object" && + value !== null && + Object.prototype.toString.call(value) === "[object ArrayBuffer]" + ); +} + +export function isArrayBufferView(value: unknown): value is ArrayBufferView { + return typeof value === "object" && value !== null && ArrayBuffer.isView(value); +} + +export function getContentName(request: Request): string | undefined; +export function getContentName(contentDisposition: string): string | undefined; +export function getContentName(headers: Headers): string | undefined; +export function getContentName(ctx: Headers | Request | string): string | undefined { + let c: string = ""; + + if (typeof ctx === "string") { + c = ctx; + } else if (ctx instanceof Headers) { + c = ctx.get("Content-Disposition") || ""; + } else if (ctx instanceof Request) { + c = ctx.headers.get("Content-Disposition") || ""; + } + + const match = c.match(/filename\*?=(?:UTF-8'')?("?)([^";]+)\1/); + return match ? match[2] : undefined; +} + +const FILE_SIGNATURES: Record = { + "89504E47": "image/png", + FFD8FF: "image/jpeg", + "47494638": "image/gif", + "49492A00": "image/tiff", // Little Endian TIFF + "4D4D002A": "image/tiff", // Big Endian TIFF + "52494646????57454250": "image/webp", // WEBP (RIFF....WEBP) + "504B0304": "application/zip", + "25504446": "application/pdf", + "00000020667479706D70": "video/mp4", + "000001BA": "video/mpeg", + "000001B3": "video/mpeg", + "1A45DFA3": "video/webm", + "4F676753": "audio/ogg", + "494433": "audio/mpeg", // MP3 with ID3 header + FFF1: "audio/aac", + FFF9: "audio/aac", + "52494646????41564920": "audio/wav", + "52494646????57415645": "audio/wave", + "52494646????415550": "audio/aiff" +}; + +async function detectMimeType( + input: ReadableStream | ArrayBuffer | ArrayBufferView | string | Blob | File | null +): Promise { + if (!input) return; + + let buffer: Uint8Array; + + if (isReadableStream(input)) { + const reader = input.getReader(); + const { value } = await reader.read(); + if (!value) return; + buffer = new Uint8Array(value); + } else if (isBlob(input) || isFile(input)) { + buffer = new Uint8Array(await input.slice(0, 12).arrayBuffer()); + } else if (isArrayBuffer(input)) { + buffer = new Uint8Array(input); + } else if (isArrayBufferView(input)) { + buffer = new Uint8Array(input.buffer, input.byteOffset, input.byteLength); + } else if (typeof input === "string") { + buffer = new TextEncoder().encode(input); + } else { + return; + } + + const hex = Array.from(buffer.slice(0, 12)) + .map((b) => b.toString(16).padStart(2, "0").toUpperCase()) + .join(""); + + for (const [signature, mime] of Object.entries(FILE_SIGNATURES)) { + const regex = new RegExp("^" + signature.replace(/\?\?/g, "..")); + if (regex.test(hex)) return mime; + } + + return; +} + +export async function blobToFile( + blob: Blob | File | unknown, + overrides: FilePropertyBag & { name?: string } = {} +): Promise { + if (isFile(blob)) return blob; + if (!isBlob(blob)) throw new Error("Not a Blob"); + + const type = !isMimeType(overrides.type, ["application/octet-stream"]) + ? overrides.type + : await detectMimeType(blob); + const ext = type ? extension(type) : ""; + const name = overrides.name || [randomString(16), ext].filter(Boolean).join("."); + + return new File([blob], name, { + type: type || guess(name), + lastModified: Date.now() + }); +} + +export async function getFileFromContext(c: Context): Promise { + const contentType = c.req.header("Content-Type") ?? "application/octet-stream"; + + if ( + contentType?.startsWith("multipart/form-data") || + contentType?.startsWith("application/x-www-form-urlencoded") + ) { + try { + const f = await c.req.formData(); + if ([...f.values()].length > 0) { + const v = [...f.values()][0]; + return await blobToFile(v); + } + } catch (e) { + console.warn("Error parsing form data", e); + } + } else { + try { + const blob = await c.req.blob(); + if (isFile(blob)) { + return blob; + } else if (isBlob(blob)) { + return await blobToFile(blob, { name: getContentName(c.req.raw), type: contentType }); + } + } catch (e) { + console.warn("Error parsing blob", e); + } + } + + throw new Error("No file found in request"); +} + +export async function getBodyFromContext(c: Context): Promise { + const contentType = c.req.header("Content-Type") ?? "application/octet-stream"; + + if ( + !contentType?.startsWith("multipart/form-data") && + !contentType?.startsWith("application/x-www-form-urlencoded") + ) { + const body = c.req.raw.body; + if (body) { + return body; + } + } + + return getFileFromContext(c); +} + +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Status +// biome-ignore lint/suspicious/noConstEnum: +export const enum HttpStatus { + // Informational responses (100–199) + CONTINUE = 100, + SWITCHING_PROTOCOLS = 101, + PROCESSING = 102, + EARLY_HINTS = 103, + + // Successful responses (200–299) + OK = 200, + CREATED = 201, + ACCEPTED = 202, + NON_AUTHORITATIVE_INFORMATION = 203, + NO_CONTENT = 204, + RESET_CONTENT = 205, + PARTIAL_CONTENT = 206, + MULTI_STATUS = 207, + ALREADY_REPORTED = 208, + IM_USED = 226, + + // Redirection messages (300–399) + MULTIPLE_CHOICES = 300, + MOVED_PERMANENTLY = 301, + FOUND = 302, + SEE_OTHER = 303, + NOT_MODIFIED = 304, + USE_PROXY = 305, + TEMPORARY_REDIRECT = 307, + PERMANENT_REDIRECT = 308, + + // Client error responses (400–499) + BAD_REQUEST = 400, + UNAUTHORIZED = 401, + PAYMENT_REQUIRED = 402, + FORBIDDEN = 403, + NOT_FOUND = 404, + METHOD_NOT_ALLOWED = 405, + NOT_ACCEPTABLE = 406, + PROXY_AUTHENTICATION_REQUIRED = 407, + REQUEST_TIMEOUT = 408, + CONFLICT = 409, + GONE = 410, + LENGTH_REQUIRED = 411, + PRECONDITION_FAILED = 412, + PAYLOAD_TOO_LARGE = 413, + URI_TOO_LONG = 414, + UNSUPPORTED_MEDIA_TYPE = 415, + RANGE_NOT_SATISFIABLE = 416, + EXPECTATION_FAILED = 417, + IM_A_TEAPOT = 418, + MISDIRECTED_REQUEST = 421, + UNPROCESSABLE_ENTITY = 422, + LOCKED = 423, + FAILED_DEPENDENCY = 424, + TOO_EARLY = 425, + UPGRADE_REQUIRED = 426, + PRECONDITION_REQUIRED = 428, + TOO_MANY_REQUESTS = 429, + REQUEST_HEADER_FIELDS_TOO_LARGE = 431, + UNAVAILABLE_FOR_LEGAL_REASONS = 451, + + // Server error responses (500–599) + INTERNAL_SERVER_ERROR = 500, + NOT_IMPLEMENTED = 501, + BAD_GATEWAY = 502, + SERVICE_UNAVAILABLE = 503, + GATEWAY_TIMEOUT = 504, + HTTP_VERSION_NOT_SUPPORTED = 505, + VARIANT_ALSO_NEGOTIATES = 506, + INSUFFICIENT_STORAGE = 507, + LOOP_DETECTED = 508, + NOT_EXTENDED = 510, + NETWORK_AUTHENTICATION_REQUIRED = 511 +} diff --git a/app/src/core/utils/test.ts b/app/src/core/utils/test.ts index 662b33c9..0b9b69ac 100644 --- a/app/src/core/utils/test.ts +++ b/app/src/core/utils/test.ts @@ -42,3 +42,21 @@ export function enableConsoleLog() { console[severity as ConsoleSeverity] = fn; }); } + +export function tryit(fn: () => void, fallback?: any) { + try { + return fn(); + } catch (e) { + return fallback || e; + } +} + +export function formatMemoryUsage() { + const usage = process.memoryUsage(); + return { + rss: usage.rss / 1024 / 1024, + heapUsed: usage.heapUsed / 1024 / 1024, + external: usage.external / 1024 / 1024, + arrayBuffers: usage.arrayBuffers / 1024 / 1024 + }; +} diff --git a/app/src/media/AppMedia.ts b/app/src/media/AppMedia.ts index dda2cf6b..4924ed91 100644 --- a/app/src/media/AppMedia.ts +++ b/app/src/media/AppMedia.ts @@ -123,7 +123,8 @@ export class AppMedia extends Module { async (e) => { const mutator = em.mutator(media); mutator.__unstable_toggleSystemEntityCreation(false); - await mutator.insertOne(this.uploadedEventDataToMediaPayload(e.params)); + const payload = this.uploadedEventDataToMediaPayload(e.params); + await mutator.insertOne(payload); mutator.__unstable_toggleSystemEntityCreation(true); console.log("App:storage:file uploaded", e); }, diff --git a/app/src/media/api/MediaApi.ts b/app/src/media/api/MediaApi.ts index 722f94de..83dd5047 100644 --- a/app/src/media/api/MediaApi.ts +++ b/app/src/media/api/MediaApi.ts @@ -1,3 +1,4 @@ +import type { FileListObject } from "media"; import { type BaseModuleApiOptions, ModuleApi, type PrimaryFieldType } from "modules/ModuleApi"; import type { FileWithPath } from "ui/elements/media/file-selector"; @@ -10,12 +11,32 @@ export class MediaApi extends ModuleApi { }; } - getFiles() { - return this.get(["files"]); + listFiles() { + return this.get(["files"]); } getFile(filename: string) { - return this.get(["file", filename]); + return this.get>(["file", filename], undefined, { + headers: { + Accept: "*/*" + } + }); + } + + async getFileStream(filename: string): Promise> { + const { res } = await this.getFile(filename); + if (!res.ok || !res.body) { + throw new Error("Failed to fetch file"); + } + return res.body; + } + + async download(filename: string): Promise { + const { res } = await this.getFile(filename); + if (!res.ok || !res.body) { + throw new Error("Failed to fetch file"); + } + return (await res.blob()) as File; } getFileUploadUrl(file: FileWithPath): string { @@ -32,10 +53,46 @@ export class MediaApi extends ModuleApi { }); } - uploadFile(file: File) { - const formData = new FormData(); - formData.append("file", file); - return this.post(["upload"], formData); + protected uploadFile(body: File | ReadableStream, filename?: string) { + let type: string = "application/octet-stream"; + let name: string = filename || ""; + try { + type = (body as File).type; + if (!filename) { + name = (body as File).name; + } + } catch (e) {} + + if (name && name.length > 0 && name.includes("/")) { + name = name.split("/").pop() || ""; + } + + if (!name || name.length === 0) { + throw new Error("Invalid filename"); + } + + return this.post(["upload", name], body, { + headers: { + "Content-Type": type + } + }); + } + + async upload(item: Request | Response | string | File | ReadableStream, filename?: string) { + if (item instanceof Request || typeof item === "string") { + const res = await this.fetcher(item); + if (!res.ok || !res.body) { + throw new Error("Failed to fetch file"); + } + return this.uploadFile(res.body, filename); + } else if (item instanceof Response) { + if (!item.body) { + throw new Error("Invalid response"); + } + return this.uploadFile(item.body, filename); + } + + return this.uploadFile(item, filename); } deleteFile(filename: string) { diff --git a/app/src/media/api/MediaController.ts b/app/src/media/api/MediaController.ts index e4698302..fc70cced 100644 --- a/app/src/media/api/MediaController.ts +++ b/app/src/media/api/MediaController.ts @@ -1,6 +1,5 @@ -import { tbValidator as tb } from "core"; -import { Type } from "core/utils"; -import { bodyLimit } from "hono/body-limit"; +import { isDebug, tbValidator as tb } from "core"; +import { HttpStatus, Type, getFileFromContext } from "core/utils"; import type { StorageAdapter } from "media"; import { StorageEvents, getRandomizedFilename } from "media"; import { Controller } from "modules/Controller"; @@ -42,7 +41,6 @@ export class MediaController extends Controller { if (!filename) { throw new Error("No file name provided"); } - //console.log("getting file", filename, headersToObject(c.req.raw.headers)); await this.getStorage().emgr.emit(new StorageEvents.FileAccessEvent({ name: filename })); return await this.getStorageAdapter().getObject(filename, c.req.raw.headers); @@ -59,24 +57,40 @@ export class MediaController extends Controller { return c.json({ message: "File deleted" }); }); - const uploadSizeMiddleware = bodyLimit({ - maxSize: this.getStorage().getConfig().body_max_size, - onError: (c: any) => { - return c.text(`Payload exceeds ${this.getStorage().getConfig().body_max_size}`, 413); - } - }); + const maxSize = this.getStorage().getConfig().body_max_size ?? Number.POSITIVE_INFINITY; + + if (isDebug()) { + hono.post("/inspect", async (c) => { + const file = await getFileFromContext(c); + return c.json({ + type: file?.type, + name: file?.name, + size: file?.size + }); + }); + } // upload file // @todo: add required type for "upload endpoints" - hono.post("/upload/:filename", uploadSizeMiddleware, async (c) => { + hono.post("/upload/:filename", async (c) => { const { filename } = c.req.param(); if (!filename) { throw new Error("No file name provided"); } - const file = await this.getStorage().getFileFromRequest(c); - console.log("----file", file); - return c.json(await this.getStorage().uploadFile(file, filename)); + const body = await getFileFromContext(c); + if (!body) { + return c.json({ error: "No file provided" }, HttpStatus.BAD_REQUEST); + } + if (body.size > maxSize) { + return c.json( + { error: `Max size (${maxSize} bytes) exceeded` }, + HttpStatus.PAYLOAD_TOO_LARGE + ); + } + + const res = await this.getStorage().uploadFile(body, filename); + return c.json(res, HttpStatus.CREATED); }); // add upload file to entity @@ -89,23 +103,21 @@ export class MediaController extends Controller { overwrite: Type.Optional(booleanLike) }) ), - uploadSizeMiddleware, async (c) => { const entity_name = c.req.param("entity"); const field_name = c.req.param("field"); const entity_id = Number.parseInt(c.req.param("id")); - console.log("params", { entity_name, field_name, entity_id }); // check if entity exists const entity = this.media.em.entity(entity_name); if (!entity) { - return c.json({ error: `Entity "${entity_name}" not found` }, 404); + return c.json({ error: `Entity "${entity_name}" not found` }, HttpStatus.NOT_FOUND); } // check if field exists and is of type MediaField const field = entity.field(field_name); if (!field || !(field instanceof MediaField)) { - return c.json({ error: `Invalid field "${field_name}"` }, 400); + return c.json({ error: `Invalid field "${field_name}"` }, HttpStatus.BAD_REQUEST); } const media_entity = this.media.getMediaEntity().name as "media"; @@ -127,7 +139,10 @@ export class MediaController extends Controller { if (count >= max_items) { // if overwrite not set, abort early if (!overwrite) { - return c.json({ error: `Max items (${max_items}) reached` }, 400); + return c.json( + { error: `Max items (${max_items}) reached` }, + HttpStatus.BAD_REQUEST + ); } // if already more in database than allowed, abort early @@ -135,7 +150,7 @@ export class MediaController extends Controller { if (count > max_items) { return c.json( { error: `Max items (${max_items}) exceeded already with ${count} items.` }, - 400 + HttpStatus.UNPROCESSABLE_ENTITY ); } @@ -161,11 +176,21 @@ export class MediaController extends Controller { if (!exists) { return c.json( { error: `Entity "${entity_name}" with ID "${entity_id}" doesn't exist found` }, - 404 + HttpStatus.NOT_FOUND + ); + } + + const file = await getFileFromContext(c); + if (!file) { + return c.json({ error: "No file provided" }, HttpStatus.BAD_REQUEST); + } + if (file.size > maxSize) { + return c.json( + { error: `Max size (${maxSize} bytes) exceeded` }, + HttpStatus.PAYLOAD_TOO_LARGE ); } - const file = await this.getStorage().getFileFromRequest(c); const file_name = getRandomizedFilename(file as File); const info = await this.getStorage().uploadFile(file, file_name, true); @@ -185,7 +210,7 @@ export class MediaController extends Controller { } } - return c.json({ ok: true, result: result.data, ...info }); + return c.json({ ok: true, result: result.data, ...info }, HttpStatus.CREATED); } ); diff --git a/app/src/media/storage/Storage.ts b/app/src/media/storage/Storage.ts index 701319a1..1a17c7bb 100644 --- a/app/src/media/storage/Storage.ts +++ b/app/src/media/storage/Storage.ts @@ -1,7 +1,5 @@ import { type EmitsEvents, EventManager } from "core/events"; -import type { TSchema } from "core/utils"; -import { type Context, Hono } from "hono"; -import { bodyLimit } from "hono/body-limit"; +import { type TSchema, isFile } from "core/utils"; import * as StorageEvents from "./events"; import type { FileUploadedEventData } from "./events"; @@ -12,7 +10,7 @@ export type FileListObject = { }; export type FileMeta = { type: string; size: number }; -export type FileBody = ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob | File; +export type FileBody = ReadableStream | File; export type FileUploadPayload = { name: string; meta: FileMeta; @@ -38,7 +36,7 @@ export interface StorageAdapter { } export type StorageConfig = { - body_max_size: number; + body_max_size?: number; }; export class Storage implements EmitsEvents { @@ -55,7 +53,7 @@ export class Storage implements EmitsEvents { this.#adapter = adapter; this.config = { ...config, - body_max_size: config.body_max_size ?? 20 * 1024 * 1024 + body_max_size: config.body_max_size }; this.emgr = emgr ?? new EventManager(); @@ -82,7 +80,6 @@ export class Storage implements EmitsEvents { noEmit?: boolean ): Promise { const result = await this.#adapter.putObject(name, file); - console.log("result", result); let info: FileUploadPayload; @@ -90,13 +87,25 @@ export class Storage implements EmitsEvents { case "undefined": throw new Error("Failed to upload file"); case "string": { - // get object meta - const meta = await this.#adapter.getObjectMeta(name); - if (!meta) { - throw new Error("Failed to get object meta"); - } + if (isFile(file)) { + info = { + name, + meta: { + size: file.size, + type: file.type + }, + etag: result + }; + break; + } else { + // get object meta + const meta = await this.#adapter.getObjectMeta(name); + if (!meta) { + throw new Error("Failed to get object meta"); + } - info = { name, meta, etag: result }; + info = { name, meta, etag: result }; + } break; } case "object": @@ -127,102 +136,4 @@ export class Storage implements EmitsEvents { async fileExists(name: string) { return await this.#adapter.objectExists(name); } - - getController(): any { - // @todo: multiple providers? - // @todo: implement range requests - - const hono = new Hono(); - - // get files list (temporary) - hono.get("/files", async (c) => { - const files = await this.#adapter.listObjects(); - return c.json(files); - }); - - // get file by name - hono.get("/file/:filename", async (c) => { - const { filename } = c.req.param(); - if (!filename) { - throw new Error("No file name provided"); - } - //console.log("getting file", filename, headersToObject(c.req.raw.headers)); - - await this.emgr.emit(new StorageEvents.FileAccessEvent({ name: filename })); - return await this.#adapter.getObject(filename, c.req.raw.headers); - }); - - // delete a file by name - hono.delete("/file/:filename", async (c) => { - const { filename } = c.req.param(); - if (!filename) { - throw new Error("No file name provided"); - } - await this.deleteFile(filename); - - return c.json({ message: "File deleted" }); - }); - - // upload file - hono.post( - "/upload/:filename", - bodyLimit({ - maxSize: this.config.body_max_size, - onError: (c: any) => { - return c.text(`Payload exceeds ${this.config.body_max_size}`, 413); - } - }), - async (c) => { - const { filename } = c.req.param(); - if (!filename) { - throw new Error("No file name provided"); - } - - const file = await this.getFileFromRequest(c); - return c.json(await this.uploadFile(file, filename)); - } - ); - - return hono; - } - - /** - * If uploaded through HttpPie -> ReadableStream - * If uploaded in tests -> file == ReadableStream - * If uploaded in FE -> content_type:body multipart/form-data; boundary=----WebKitFormBoundary7euoBFF12B0AHWLn - * file File { - * size: 223052, - * type: 'image/png', - * name: 'noise_white.png', - * lastModified: 1731743671176 - * } - * @param c - */ - async getFileFromRequest(c: Context): Promise { - const content_type = c.req.header("Content-Type") ?? "application/octet-stream"; - console.log("content_type:body", content_type); - const body = c.req.raw.body; - if (!body) { - throw new Error("No body"); - } - - let file: FileBody | undefined; - if (content_type?.startsWith("multipart/form-data")) { - file = (await c.req.formData()).get("file") as File; - // @todo: check nextjs, it's not *that* [File] type (but it's uploadable) - if (typeof file === "undefined") { - throw new Error("No file given at form data 'file'"); - } - /*console.log("file", file); - if (!(file instanceof File)) { - throw new Error("No file given at form data 'file'"); - }*/ - } else if (content_type?.startsWith("application/octet-stream")) { - file = body; - } else { - throw new Error(`Unsupported content type: ${content_type}`); - } - - return file; - } } diff --git a/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts b/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts index cfb4100c..7f2de8c6 100644 --- a/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts +++ b/app/src/media/storage/adapters/StorageCloudinaryAdapter.ts @@ -13,12 +13,6 @@ export const cloudinaryAdapterConfig = Type.Object( ); export type CloudinaryConfig = Static; -/*export type CloudinaryConfig = { - cloud_name: string; - api_key: string; - api_secret: string; - upload_preset?: string; -};*/ type CloudinaryObject = { asset_id: string; @@ -91,10 +85,8 @@ export class StorageCloudinaryAdapter implements StorageAdapter { } async putObject(_key: string, body: FileBody) { - //console.log("_key", _key); // remove extension, as it is added by cloudinary const key = _key.replace(/\.[a-z0-9]{2,5}$/, ""); - //console.log("key", key); const formData = new FormData(); formData.append("file", body as any); @@ -117,21 +109,12 @@ export class StorageCloudinaryAdapter implements StorageAdapter { body: formData } ); - //console.log("putObject:cloudinary", formData); if (!result.ok) { - /*console.log( - "failed to upload using cloudinary", - Object.fromEntries(formData.entries()), - result - );*/ return undefined; } - //console.log("putObject:result", result); - const data = (await result.json()) as CloudinaryPutObjectResponse; - //console.log("putObject:result:json", data); return { name: data.public_id + "." + data.format, @@ -154,7 +137,6 @@ export class StorageCloudinaryAdapter implements StorageAdapter { } } ); - //console.log("result", result); if (!result.ok) { throw new Error("Failed to list objects"); @@ -179,10 +161,7 @@ export class StorageCloudinaryAdapter implements StorageAdapter { } async objectExists(key: string): Promise { - //console.log("--object exists?", key); const result = await this.headObject(key); - //console.log("object exists", result); - return result.ok; } @@ -214,12 +193,10 @@ export class StorageCloudinaryAdapter implements StorageAdapter { const type = this.guessType(key) ?? "image"; const objectUrl = `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/${key}`; - //console.log("objectUrl", objectUrl); return objectUrl; } async getObject(key: string, headers: Headers): Promise { - //console.log("url", this.getObjectUrl(key)); const res = await fetch(this.getObjectUrl(key), { method: "GET", headers: pickHeaders(headers, ["range"]) @@ -237,14 +214,10 @@ export class StorageCloudinaryAdapter implements StorageAdapter { const formData = new FormData(); formData.append("public_ids[]", key); - const result = await fetch( - `https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, - { - method: "DELETE", - body: formData - } - ); - //console.log("deleteObject:result", result); + await fetch(`https://res.cloudinary.com/${this.config.cloud_name}/${type}/upload/`, { + method: "DELETE", + body: formData + }); } toJSON(secrets?: boolean) { diff --git a/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts b/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts index 2c142ff8..8b2f9ba2 100644 --- a/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts +++ b/app/src/media/storage/adapters/StorageLocalAdapter/StorageLocalAdapter.ts @@ -1,6 +1,12 @@ import { readFile, readdir, stat, unlink, writeFile } from "node:fs/promises"; -import { type Static, Type, parse } from "core/utils"; -import type { FileBody, FileListObject, FileMeta, StorageAdapter } from "../../Storage"; +import { type Static, Type, isFile, parse } from "core/utils"; +import type { + FileBody, + FileListObject, + FileMeta, + FileUploadPayload, + StorageAdapter +} from "../../Storage"; import { guess } from "../../mime-types-tiny"; export const localAdapterConfig = Type.Object( @@ -43,8 +49,9 @@ export class StorageLocalAdapter implements StorageAdapter { return fileStats; } - private async computeEtag(content: BufferSource): Promise { - const hashBuffer = await crypto.subtle.digest("SHA-256", content); + private async computeEtag(body: FileBody): Promise { + const content = isFile(body) ? body : new Response(body); + const hashBuffer = await crypto.subtle.digest("SHA-256", await content.arrayBuffer()); const hashArray = Array.from(new Uint8Array(hashBuffer)); const hashHex = hashArray.map((byte) => byte.toString(16).padStart(2, "0")).join(""); @@ -52,17 +59,16 @@ export class StorageLocalAdapter implements StorageAdapter { return `"${hashHex}"`; } - async putObject(key: string, body: FileBody): Promise { + async putObject(key: string, body: FileBody): Promise { if (body === null) { throw new Error("Body is empty"); } - // @todo: this is too hacky - const file = body as File; - const filePath = `${this.config.path}/${key}`; - await writeFile(filePath, file.stream()); - return await this.computeEtag(await file.arrayBuffer()); + const is_file = isFile(body); + await writeFile(filePath, is_file ? body.stream() : body); + + return await this.computeEtag(body); } async deleteObject(key: string): Promise { diff --git a/app/src/media/storage/adapters/StorageS3Adapter.ts b/app/src/media/storage/adapters/StorageS3Adapter.ts index b330d645..cd051d7e 100644 --- a/app/src/media/storage/adapters/StorageS3Adapter.ts +++ b/app/src/media/storage/adapters/StorageS3Adapter.ts @@ -7,7 +7,7 @@ import type { PutObjectRequest } from "@aws-sdk/client-s3"; import { AwsClient, isDebug } from "core"; -import { type Static, Type, parse, pickHeaders } from "core/utils"; +import { type Static, Type, isFile, parse, pickHeaders } from "core/utils"; import { transform } from "lodash-es"; import type { FileBody, FileListObject, StorageAdapter } from "../Storage"; @@ -82,17 +82,14 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter { }; const url = this.getUrl("", params); - //console.log("url", url); const res = await this.fetchJson<{ ListBucketResult: ListObjectsV2Output }>(url, { method: "GET" }); - //console.log("res", res); // absolutely weird, but if only one object is there, it's an object, not an array const { Contents } = res.ListBucketResult; const objects = !Contents ? [] : Array.isArray(Contents) ? Contents : [Contents]; - //console.log(JSON.stringify(res.ListBucketResult, null, 2), objects); const transformed = transform( objects, (acc, obj) => { @@ -107,28 +104,21 @@ export class StorageS3Adapter extends AwsClient implements StorageAdapter { }, [] as FileListObject[] ); - //console.log(transformed); return transformed; } async putObject( key: string, - body: FileBody | null, + body: FileBody, // @todo: params must be added as headers, skipping for now params: Omit = {} ) { const url = this.getUrl(key, {}); - //console.log("url", url); const res = await this.fetch(url, { method: "PUT", body }); - /*console.log("putObject:raw:res", { - ok: res.ok, - status: res.status, - statusText: res.statusText, - });*/ if (res.ok) { // "df20fcb574dba1446cf5ec997940492b" diff --git a/app/src/media/storage/mime-types-tiny.ts b/app/src/media/storage/mime-types-tiny.ts index a2317342..f54b51ef 100644 --- a/app/src/media/storage/mime-types-tiny.ts +++ b/app/src/media/storage/mime-types-tiny.ts @@ -75,3 +75,21 @@ export function guess(f: string): string { return c.a(); } } + +export function isMimeType(mime: any, exclude: string[] = []) { + for (const [k, v] of M.entries()) { + if (v === mime && !exclude.includes(k)) { + return true; + } + } + return false; +} + +export function extension(mime: string) { + for (const [k, v] of M.entries()) { + if (v === mime) { + return k; + } + } + return ""; +} diff --git a/app/src/modules/ModuleApi.ts b/app/src/modules/ModuleApi.ts index a0881707..3ba05523 100644 --- a/app/src/modules/ModuleApi.ts +++ b/app/src/modules/ModuleApi.ts @@ -23,10 +23,14 @@ export type ApiResponse = { export type TInput = string | (string | number | PrimaryFieldType)[]; export abstract class ModuleApi { + protected fetcher: typeof fetch; + constructor( protected readonly _options: Partial = {}, - protected fetcher?: typeof fetch - ) {} + fetcher?: typeof fetch + ) { + this.fetcher = fetcher ?? fetch; + } protected getDefaultOptions(): Partial { return {}; @@ -76,7 +80,9 @@ export abstract class ModuleApi( const actualData = data ?? (body as unknown as Data); const _props = ["raw", "body", "ok", "status", "res", "data", "toJSON"]; - return new Proxy(actualData as any, { + if (typeof actualData !== "object") { + throw new Error(`Response data must be an object, "${typeof actualData}" given.`); + } + + return new Proxy(actualData ?? ({} as any), { get(target, prop, receiver) { if (prop === "raw" || prop === "res") return raw; if (prop === "body") return body; @@ -228,6 +238,8 @@ export class FetchPromise> implements Promise { } } else if (contentType.startsWith("text")) { resBody = await res.text(); + } else { + resBody = res.body; } return createResponseProxy(res, resBody, resData); diff --git a/app/src/ui/components/form/json-schema-form/Form.tsx b/app/src/ui/components/form/json-schema-form/Form.tsx index c26cb01c..0cc5e979 100644 --- a/app/src/ui/components/form/json-schema-form/Form.tsx +++ b/app/src/ui/components/form/json-schema-form/Form.tsx @@ -225,12 +225,12 @@ export function FormContextOverride({ // especially useful for AnyOf, since it doesn't need to fully validate (e.g. pattern) if (prefix) { additional.root = prefix; - additional.setValue = (pointer: string, value: any) => { - ctx.setValue(prefixPointer(pointer, prefix), value); - }; - additional.deleteValue = (pointer: string) => { - ctx.deleteValue(prefixPointer(pointer, prefix)); + /*additional.setValue = (path: string, value: any) => { + ctx.setValue(prefixPath(path, prefix), value); }; + additional.deleteValue = (path: string) => { + ctx.deleteValue(prefixPath(path, prefix)); + };*/ } const context = { diff --git a/app/src/ui/elements/media/Dropzone.tsx b/app/src/ui/elements/media/Dropzone.tsx index 1a0e2d6c..537cfe0c 100644 --- a/app/src/ui/elements/media/Dropzone.tsx +++ b/app/src/ui/elements/media/Dropzone.tsx @@ -58,6 +58,15 @@ export type DropzoneProps = { children?: (props: DropzoneRenderProps) => JSX.Element; }; +function handleUploadError(e: unknown) { + if (e && e instanceof XMLHttpRequest) { + const res = JSON.parse(e.responseText) as any; + alert(`Upload failed with code ${e.status}: ${res.error}`); + } else { + alert("Upload failed"); + } +} + export function Dropzone({ getUploadInfo, handleDelete, @@ -164,7 +173,11 @@ export function Dropzone({ return; } else { for (const file of pendingFiles) { - await uploadFileProgress(file); + try { + await uploadFileProgress(file); + } catch (e) { + handleUploadError(e); + } } setUploading(false); onUploaded?.(files); @@ -226,8 +239,6 @@ export function Dropzone({ const uploadInfo = getUploadInfo(file.body); console.log("dropzone:uploadInfo", uploadInfo); const { url, headers, method = "POST" } = uploadInfo; - const formData = new FormData(); - formData.append("file", file.body); const xhr = new XMLHttpRequest(); console.log("xhr:url", url); @@ -260,7 +271,7 @@ export function Dropzone({ xhr.onload = () => { console.log("onload", file.path, xhr.status); - if (xhr.status === 200) { + if (xhr.status >= 200 && xhr.status < 300) { //setFileState(file.path, "uploaded", 1); console.log("Upload complete"); @@ -281,8 +292,8 @@ export function Dropzone({ resolve(); } else { setFileState(file.path, "failed", 1); - console.error("Upload failed with status: ", xhr.status); - reject(); + console.error("Upload failed with status: ", xhr.status, xhr.statusText); + reject(xhr); } }; @@ -295,7 +306,7 @@ export function Dropzone({ }; xhr.setRequestHeader("Accept", "application/json"); - xhr.send(formData); + xhr.send(file.body); }); } @@ -366,6 +377,14 @@ const DropzoneInner = ({ ); + async function uploadHandler(file: FileState) { + try { + return await uploadFile(file); + } catch (e) { + handleUploadError(e); + } + } + return (
))} @@ -452,6 +471,7 @@ const Preview: React.FC = ({ file, handleUpload, handleDelete }) =
diff --git a/app/src/ui/routes/test/tests/json-schema-form3.tsx b/app/src/ui/routes/test/tests/json-schema-form3.tsx index 06408724..eb0dc65f 100644 --- a/app/src/ui/routes/test/tests/json-schema-form3.tsx +++ b/app/src/ui/routes/test/tests/json-schema-form3.tsx @@ -46,7 +46,7 @@ export default function JsonSchemaForm3() { return (
-
console.log("change", data)} onSubmit={(data) => console.log("submit", data)} schema={{ @@ -68,7 +68,7 @@ export default function JsonSchemaForm3() { className="flex flex-col gap-3" validateOn="change" options={{ debug: true }} - /> + />*/} {/**/} {/**/} - {/**/} + options={{ debug: true }} + /*validateOn="change"*/ + /> {/*