experimenting with config history

Added optional `fields_order` to the entity configuration schema, allowing for custom field sorting. Updated `getFields` method to support sorting based on this configuration. Adjusted related methods and UI components to handle field order during updates and serialization. Improved handling of virtual fields in various contexts.

app/src/data/entities/Entity.ts

@@ -18,6 +18,7 @@ export const entityConfigSchema = s
          sort_field: s.string({ default: config.data.default_primary_field }),
          sort_dir: s.string({ enum: ["asc", "desc"], default: "asc" }),
          primary_format: s.string({ enum: primaryFieldTypes }),
+         fields_order: s.array(s.string()).optional(),
       },
       { default: {} },
    )
@@ -137,7 +138,9 @@ export class Entity<
    }
 
    getFillableFields(context?: TActionContext, include_virtual?: boolean): Field[] {
-      return this.getFields(include_virtual).filter((field) => field.isFillable(context));
+      return this.getFields({ virtual: include_virtual }).filter((field) =>
+         field.isFillable(context),
+      );
    }
 
    getRequiredFields(): Field[] {
@@ -189,9 +192,24 @@ export class Entity<
       return this.fields.findIndex((field) => field.name === name) !== -1;
    }
 
-   getFields(include_virtual: boolean = false): Field[] {
+   getFields(opts?: { virtual?: boolean; sorted?: boolean }): Field[] {
-      if (include_virtual) return this.fields;
+      const fields = [...this.fields];
-      return this.fields.filter((f) => !f.isVirtual());
+      const sort = opts?.sorted !== false;
+      const fields_order = this.config.fields_order;
+      if (fields_order && fields_order.length > 0 && sort) {
+         if (fields_order.length !== this.fields.length) {
+            $console.warn("Fields order must contain all fields");
+         } else {
+            // sort fields by order
+            fields.sort((a, b) => {
+               const a_index = fields_order.indexOf(a.name);
+               const b_index = fields_order.indexOf(b.name);
+               return a_index - b_index;
+            });
+         }
+      }
+      if (opts?.virtual) return fields;
+      return fields.filter((f) => !f.isVirtual());
    }
 
    addField(field: Field) {
@@ -275,7 +293,7 @@ export class Entity<
             fields = this.getFillableFields(options.context);
             break;
          default:
-            fields = this.getFields(true);
+            fields = this.getFields({ virtual: true });
       }
 
       const _fields = Object.fromEntries(fields.map((field) => [field.name, field]));
@@ -309,7 +327,12 @@ export class Entity<
    toJSON() {
       return {
          type: this.type,
-         fields: Object.fromEntries(this.fields.map((field) => [field.name, field.toJSON()])),
+         fields: Object.fromEntries(
+            this.getFields({ virtual: true, sorted: true }).map((field) => [
+               field.name,
+               field.toJSON(),
+            ]),
+         ),
          config: this.config,
       };
    }

app/src/data/schema/SchemaManager.ts

@@ -76,7 +76,7 @@ export class SchemaManager {
    }
 
    getIntrospectionFromEntity(entity: Entity): IntrospectedTable {
-      const fields = entity.getFields(false);
+      const fields = entity.getFields({ virtual: false, sorted: true });
       const indices = this.em.getIndicesOf(entity);
 
       // this is intentionally setting values to defaults, like "nullable" and "default"

app/src/modules/server/SystemController.ts

@@ -32,6 +32,7 @@ import { getVersion } from "core/env";
 import type { Module } from "modules/Module";
 import { getSystemMcp } from "modules/mcp/system-mcp";
 import type { DbModuleManager } from "modules/db/DbModuleManager";
+import type { Repository } from "data/entities";
 
 export type ConfigUpdate<Key extends ModuleKey = ModuleKey> = {
    success: true;
@@ -129,6 +130,25 @@ export class SystemController extends Controller {
             },
          );
 
+         hono.get("/history", async (c) => {
+            // @ts-expect-error "repo" is private
+            const repo = manager.repo() as Repository;
+            const { data } = await repo.findMany({
+               where: { type: "diff", version: manager.version() },
+               sort: { by: "id", dir: "desc" },
+            });
+            return c.json(data);
+         });
+
+         hono.get("/history/:id", async (c) => {
+            // @ts-expect-error "repo" is private
+            const repo = manager.repo() as Repository;
+            const { data } = await repo.findId(c.req.param("id"), {
+               where: { type: "diff", version: manager.version() },
+            });
+            return c.json(data);
+         });
+
          async function handleConfigUpdateResponse(
             c: Context<any>,
             cb: () => Promise<ConfigUpdate>,

app/src/ui/client/schema/data/use-bknd-data.ts

@@ -120,17 +120,20 @@ function entityFieldActions(bkndActions: TSchemaActions, entityName: string) {
          return await bkndActions.add("data", `entities.${entityName}.fields.${name}`, validated);
       },
       patch: () => null,
-      set: async (fields: TAppDataEntityFields) => {
+      set: async (fields: TAppDataEntityFields, order?: string[]) => {
          try {
             const validated = parse(entityFields, fields, {
                skipMark: true,
                forceParse: true,
             });
-            const res = await bkndActions.overwrite(
+            await bkndActions.overwrite("data", `entities.${entityName}.fields`, validated);
-               "data",
+            if (order) {
-               `entities.${entityName}.fields`,
+               await bkndActions.overwrite(
-               validated,
+                  "data",
-            );
+                  `entities.${entityName}.config.fields_order`,
+                  order,
+               );
+            }
          } catch (e) {
             console.error("error", e);
             if (e instanceof InvalidSchemaError) {

app/src/ui/components/table/DataTable.tsx

@@ -159,7 +159,7 @@ export function DataTable<Data extends Record<string, any> = Record<string, any>
                               )}
 
                               {Object.entries(row).map(([key, value], index) => (
-                                 <td key={index} onClick={rowClick}>
+                                 <td key={index} onClick={rowClick} valign="top">
                                     <div className="flex flex-row items-start py-3 px-3.5 font-normal ">
                                        <CellRender property={key} value={value} />
                                     </div>

app/src/ui/routes/data/data.schema.$entity.tsx

@@ -155,18 +155,25 @@ const Fields = ({ entity }: { entity: Entity }) => {
    const { readonly } = useBknd();
    const [res, setRes] = useState<any>();
    const ref = useRef<EntityFieldsFormRef>(null);
+
+   // @todo: the return of toJSON from Fields doesn't match "type" enum
+   const initialFields = Object.fromEntries(entity.fields.map((f) => [f.name, f.toJSON()])) as any;
+
    async function handleUpdate() {
       if (submitting) return;
       setSubmitting(true);
       const fields = ref.current?.getData()!;
+
       await actions.entity.patch(entity.name).fields.set(fields);
+
+      // check if field order has changed
+      if (Object.keys(initialFields).join(",") !== Object.keys(fields).join(",")) {
+         await actions.entity.patch(entity.name).fields.set(fields, Object.keys(fields));
+      }
       setSubmitting(false);
       setUpdates((u) => u + 1);
    }
 
-   // @todo: the return of toJSON from Fields doesn't match "type" enum
-   const initialFields = Object.fromEntries(entity.fields.map((f) => [f.name, f.toJSON()])) as any;
-
    return (
       <AppShell.RouteAwareSectionHeaderAccordionItem
          identifier="fields"

app/src/ui/routes/settings/index.tsx

@@ -13,6 +13,7 @@ import { DataSettings } from "./routes/data.settings";
 import { FlowsSettings } from "./routes/flows.settings";
 import { ServerSettings } from "./routes/server.settings";
 import { IconButton } from "ui/components/buttons/IconButton";
+import { SettingsHistory } from "ui/routes/settings/routes/history";
 
 function SettingsSidebar() {
    const { version, schema, actions, app } = useBknd();
@@ -75,6 +76,7 @@ export default function SettingsRoutes() {
                      />
                   )}
                />
+               <Route path="/history" nest component={SettingsHistory} />
 
                <SettingRoutesRoutes />
 

app/src/ui/routes/settings/routes/history.tsx

@@ -0,0 +1,152 @@
+import { AppShell } from "ui/layouts/AppShell";
+import { Route, Switch, useParams } from "wouter";
+import { useEffect, useState } from "react";
+import { CellValue, DataTable } from "ui/components/table/DataTable";
+import { twMerge } from "tailwind-merge";
+import { JsonViewer } from "ui/components/code/JsonViewer";
+import { useNavigate } from "ui/lib/routes";
+import { Breadcrumbs2 } from "ui/layouts/AppShell/Breadcrumbs2";
+
+export function SettingsHistory() {
+   return (
+      <Switch>
+         <Route path="/" component={SettingsHistoryList} />
+         <Route path="/:id" component={SettingsHistoryDetail} />
+      </Switch>
+   );
+}
+
+function SettingsHistoryList() {
+   const [history, setHistory] = useState<any[]>([]);
+   const [navigate] = useNavigate();
+
+   useEffect(() => {
+      fetch("/api/system/config/history")
+         .then((res) => res.json())
+         .then((data: any) =>
+            data.map((item) => ({
+               id: item.id,
+               timestamp: item.created_at,
+               actions: Array.from(new Set(item.json.map((j) => j.t))),
+               paths: Array.from(new Set(item.json.map((j) => j.p))),
+            })),
+         )
+         .then(setHistory);
+   }, []);
+
+   const onClickRow = (row) => {
+      navigate(`/${row.id}`);
+   };
+
+   return (
+      <>
+         <AppShell.SectionHeader>History</AppShell.SectionHeader>
+         <AppShell.Scrollable>
+            <div className="flex flex-col flex-grow p-3 gap-1">
+               <DataTable
+                  data={history}
+                  renderValue={renderValue}
+                  perPage={50}
+                  onClickRow={onClickRow}
+               />
+            </div>
+         </AppShell.Scrollable>
+      </>
+   );
+}
+
+const Labels = ({
+   items = [],
+   max = 3,
+   wrap = false,
+}: { items: string[]; max?: number; wrap?: boolean }) => {
+   const count = items.length;
+   if (count > max) {
+      items = [...items.slice(0, max), `+${count - max}`];
+   }
+
+   return (
+      <div className={twMerge("flex gap-1", wrap ? "flex-col items-start" : "flex-row")}>
+         {items.map((p, i) => (
+            <span
+               key={i}
+               className="inline-block px-2 py-1.5 text-sm bg-primary/5 rounded font-mono leading-none"
+            >
+               {p}
+            </span>
+         ))}
+      </div>
+   );
+};
+
+const renderValue = ({ value, property }) => {
+   if (property === "timestamp") {
+      return <span>{new Date(value).toLocaleString()}</span>;
+   }
+
+   if (property === "actions") {
+      return (
+         <Labels
+            items={value.map((a) => {
+               switch (a) {
+                  case "a":
+                     return "Add";
+                  case "r":
+                     return "Remove";
+                  case "e":
+                     return "Edit";
+                  default:
+                     return "Unknown";
+               }
+            })}
+         />
+      );
+   }
+
+   if (property === "paths") {
+      return <Labels wrap items={value.map((p) => p.join("."))} />;
+   }
+
+   return <CellValue value={value} property={property} />;
+};
+
+function SettingsHistoryDetail() {
+   const { id } = useParams();
+   const [item, setItem] = useState<any>();
+
+   useEffect(() => {
+      fetch(`/api/system/config/history/${id}`)
+         .then((res) => res.json())
+         .then(setItem);
+   }, []);
+
+   return (
+      <>
+         <AppShell.SectionHeader className="pl-3">
+            <Breadcrumbs2 path={[{ label: "History", href: "/" }, { label: `#${id}` }]} />
+         </AppShell.SectionHeader>
+         <AppShell.Scrollable>
+            <div className="flex flex-col flex-grow p-3 gap-1">
+               {item?.json?.map((item, i) => (
+                  <DiffItem key={i} item={item} />
+               ))}
+            </div>
+         </AppShell.Scrollable>
+      </>
+   );
+}
+
+const DiffItem = ({ item }: { item: any }) => {
+   return (
+      <div className="flex flex-col gap-1 w-full border-b border-muted p-3">
+         <div className="flex flex-row gap-1 w-full">
+            <span className="font-mono">{item.t}</span>
+            <span className="font-mono">{item.p.join(".")}</span>
+         </div>
+         <div className="flex flex-row gap-1 w-full">
+            <JsonViewer json={item.o} expand={10} className="w-1/2" title="Old" />
+            <JsonViewer json={item.n} expand={10} className="w-1/2" title="New" />
+         </div>
+      </div>
+   );
+};