From a1084f91e56677b1cae3bb35e8f9f4cd0630dc81 Mon Sep 17 00:00:00 2001 From: Meier Lukas Date: Fri, 17 Jan 2025 12:53:01 +0100 Subject: [PATCH] fix: mysql transactions do not work with run property of sqlite (#1974) * fix: mysql transactions do not work with run property of sqlite * fix: ci issues --- packages/api/src/router/board.ts | 653 ++++++++++++------ .../router/integration/integration-router.ts | 108 ++- packages/cron-jobs/src/jobs/icons-updater.ts | 108 ++- packages/db/driver.ts | 4 +- packages/db/index.ts | 2 + packages/db/transactions.ts | 24 + .../src/import/collections/common.ts | 12 +- .../src/import/import-initial-oldmarr.ts | 21 +- .../src/import/import-single-oldmarr.ts | 12 +- 9 files changed, 677 insertions(+), 267 deletions(-) create mode 100644 packages/db/transactions.ts diff --git a/packages/api/src/router/board.ts b/packages/api/src/router/board.ts index 1fc1b9abf..72ae2bfd2 100644 --- a/packages/api/src/router/board.ts +++ b/packages/api/src/router/board.ts @@ -4,7 +4,7 @@ import superjson from "superjson"; import { constructBoardPermissions } from "@homarr/auth/shared"; import type { DeviceType } from "@homarr/common/server"; import type { Database, InferInsertModel, InferSelectModel, SQL } from "@homarr/db"; -import { and, createId, eq, inArray, like, or } from "@homarr/db"; +import { and, createId, eq, handleTransactionsAsync, inArray, like, or } from "@homarr/db"; import { getServerSettingByKeyAsync } from "@homarr/db/queries"; import { boardGroupPermissions, @@ -204,21 +204,49 @@ export const boardRouter = createTRPCRouter({ .input(validation.board.create) .mutation(async ({ ctx, input }) => { const boardId = createId(); - await ctx.db.transaction(async (transaction) => { - await transaction.insert(boards).values({ - id: boardId, - name: input.name, - isPublic: input.isPublic, - columnCount: input.columnCount, - creatorId: ctx.session.user.id, - }); - await transaction.insert(sections).values({ - id: createId(), - kind: "empty", - xOffset: 0, - yOffset: 0, - boardId, - }); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + await transaction.insert(schema.boards).values({ + id: boardId, + name: input.name, + isPublic: input.isPublic, + columnCount: input.columnCount, + creatorId: ctx.session.user.id, + }); + await transaction.insert(schema.sections).values({ + id: createId(), + kind: "empty", + xOffset: 0, + yOffset: 0, + boardId, + }); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction + .insert(boards) + .values({ + id: boardId, + name: input.name, + isPublic: input.isPublic, + columnCount: input.columnCount, + creatorId: ctx.session.user.id, + }) + .run(); + transaction + .insert(sections) + .values({ + id: createId(), + kind: "empty", + xOffset: 0, + yOffset: 0, + boardId, + }) + .run(); + }); + }, }); }), duplicateBoard: permissionRequiredProcedure @@ -302,28 +330,54 @@ export const boardRouter = createTRPCRouter({ })), ); - ctx.db.transaction((transaction) => { - transaction - .insert(boards) - .values({ - ...boardProps, - id: newBoardId, - name: input.name, - creatorId: ctx.session.user.id, - }) - .run(); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + transaction.insert(schema.boards).values({ + ...boardProps, + id: newBoardId, + name: input.name, + creatorId: ctx.session.user.id, + }); - if (sectionsToInsert.length > 0) { - transaction.insert(sections).values(sectionsToInsert).run(); - } + if (sectionsToInsert.length > 0) { + await transaction.insert(schema.sections).values(sectionsToInsert); + } - if (itemsToInsert.length > 0) { - transaction.insert(items).values(itemsToInsert).run(); - } + if (itemsToInsert.length > 0) { + await transaction.insert(schema.items).values(itemsToInsert); + } - if (itemIntegrationsToInsert.length > 0) { - transaction.insert(integrationItems).values(itemIntegrationsToInsert).run(); - } + if (itemIntegrationsToInsert.length > 0) { + await transaction.insert(schema.integrationItems).values(itemIntegrationsToInsert); + } + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction + .insert(boards) + .values({ + ...boardProps, + id: newBoardId, + name: input.name, + creatorId: ctx.session.user.id, + }) + .run(); + + if (sectionsToInsert.length > 0) { + transaction.insert(sections).values(sectionsToInsert).run(); + } + + if (itemsToInsert.length > 0) { + transaction.insert(items).values(itemsToInsert).run(); + } + + if (itemIntegrationsToInsert.length > 0) { + transaction.insert(integrationItems).values(itemIntegrationsToInsert).run(); + } + }); + }, }); }), renameBoard: protectedProcedure.input(validation.board.rename).mutation(async ({ ctx, input }) => { @@ -434,148 +488,313 @@ export const boardRouter = createTRPCRouter({ saveBoard: protectedProcedure.input(validation.board.save).mutation(async ({ input, ctx }) => { await throwIfActionForbiddenAsync(ctx, eq(boards.id, input.id), "modify"); - await ctx.db.transaction(async (transaction) => { - const dbBoard = await getFullBoardWithWhereAsync(transaction, eq(boards.id, input.id), ctx.session.user.id); + const dbBoard = await getFullBoardWithWhereAsync(ctx.db, eq(boards.id, input.id), ctx.session.user.id); - const addedSections = filterAddedItems(input.sections, dbBoard.sections); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + const addedSections = filterAddedItems(input.sections, dbBoard.sections); - if (addedSections.length > 0) { - await transaction.insert(sections).values( - addedSections.map((section) => ({ - id: section.id, - kind: section.kind, - yOffset: section.yOffset, - xOffset: section.kind === "dynamic" ? section.xOffset : 0, - height: "height" in section ? section.height : null, - width: "width" in section ? section.width : null, - parentSectionId: "parentSectionId" in section ? section.parentSectionId : null, - name: "name" in section ? section.name : null, - boardId: dbBoard.id, - })), - ); - } + if (addedSections.length > 0) { + await transaction.insert(schema.sections).values( + addedSections.map((section) => ({ + id: section.id, + kind: section.kind, + yOffset: section.yOffset, + xOffset: section.kind === "dynamic" ? section.xOffset : 0, + height: "height" in section ? section.height : null, + width: "width" in section ? section.width : null, + parentSectionId: "parentSectionId" in section ? section.parentSectionId : null, + name: "name" in section ? section.name : null, + boardId: dbBoard.id, + })), + ); + } - const inputItems = input.sections.flatMap((section) => - section.items.map((item) => ({ ...item, sectionId: section.id })), - ); - const dbItems = dbBoard.sections.flatMap((section) => - section.items.map((item) => ({ ...item, sectionId: section.id })), - ); - - const addedItems = filterAddedItems(inputItems, dbItems); - - if (addedItems.length > 0) { - await transaction.insert(items).values( - addedItems.map((item) => ({ - id: item.id, - kind: item.kind, - height: item.height, - width: item.width, - xOffset: item.xOffset, - yOffset: item.yOffset, - options: superjson.stringify(item.options), - advancedOptions: superjson.stringify(item.advancedOptions), - sectionId: item.sectionId, - })), - ); - } - - const inputIntegrationRelations = inputItems.flatMap(({ integrationIds, id: itemId }) => - integrationIds.map((integrationId) => ({ - integrationId, - itemId, - })), - ); - const dbIntegrationRelations = dbItems.flatMap(({ integrationIds, id: itemId }) => - integrationIds.map((integrationId) => ({ - integrationId, - itemId, - })), - ); - const addedIntegrationRelations = inputIntegrationRelations.filter( - (inputRelation) => - !dbIntegrationRelations.some( - (dbRelation) => - dbRelation.itemId === inputRelation.itemId && dbRelation.integrationId === inputRelation.integrationId, - ), - ); - - if (addedIntegrationRelations.length > 0) { - await transaction.insert(integrationItems).values( - addedIntegrationRelations.map((relation) => ({ - itemId: relation.itemId, - integrationId: relation.integrationId, - })), - ); - } - - const updatedItems = filterUpdatedItems(inputItems, dbItems); - - for (const item of updatedItems) { - await transaction - .update(items) - .set({ - kind: item.kind, - height: item.height, - width: item.width, - xOffset: item.xOffset, - yOffset: item.yOffset, - options: superjson.stringify(item.options), - advancedOptions: superjson.stringify(item.advancedOptions), - sectionId: item.sectionId, - }) - .where(eq(items.id, item.id)); - } - - const updatedSections = filterUpdatedItems(input.sections, dbBoard.sections); - - for (const section of updatedSections) { - const prev = dbBoard.sections.find((dbSection) => dbSection.id === section.id); - await transaction - .update(sections) - .set({ - yOffset: section.yOffset, - xOffset: section.xOffset, - height: prev?.kind === "dynamic" && "height" in section ? section.height : null, - width: prev?.kind === "dynamic" && "width" in section ? section.width : null, - parentSectionId: prev?.kind === "dynamic" && "parentSectionId" in section ? section.parentSectionId : null, - name: prev?.kind === "category" && "name" in section ? section.name : null, - }) - .where(eq(sections.id, section.id)); - } - - const removedIntegrationRelations = dbIntegrationRelations.filter( - (dbRelation) => - !inputIntegrationRelations.some( - (inputRelation) => - dbRelation.itemId === inputRelation.itemId && dbRelation.integrationId === inputRelation.integrationId, - ), - ); - - for (const relation of removedIntegrationRelations) { - await transaction - .delete(integrationItems) - .where( - and( - eq(integrationItems.itemId, relation.itemId), - eq(integrationItems.integrationId, relation.integrationId), - ), + const inputItems = input.sections.flatMap((section) => + section.items.map((item) => ({ ...item, sectionId: section.id })), + ); + const dbItems = dbBoard.sections.flatMap((section) => + section.items.map((item) => ({ ...item, sectionId: section.id })), ); - } - const removedItems = filterRemovedItems(inputItems, dbItems); + const addedItems = filterAddedItems(inputItems, dbItems); - const itemIds = removedItems.map((item) => item.id); - if (itemIds.length > 0) { - await transaction.delete(items).where(inArray(items.id, itemIds)); - } + if (addedItems.length > 0) { + await transaction.insert(schema.items).values( + addedItems.map((item) => ({ + id: item.id, + kind: item.kind, + height: item.height, + width: item.width, + xOffset: item.xOffset, + yOffset: item.yOffset, + options: superjson.stringify(item.options), + advancedOptions: superjson.stringify(item.advancedOptions), + sectionId: item.sectionId, + })), + ); + } - const removedSections = filterRemovedItems(input.sections, dbBoard.sections); - const sectionIds = removedSections.map((section) => section.id); + const inputIntegrationRelations = inputItems.flatMap(({ integrationIds, id: itemId }) => + integrationIds.map((integrationId) => ({ + integrationId, + itemId, + })), + ); + const dbIntegrationRelations = dbItems.flatMap(({ integrationIds, id: itemId }) => + integrationIds.map((integrationId) => ({ + integrationId, + itemId, + })), + ); + const addedIntegrationRelations = inputIntegrationRelations.filter( + (inputRelation) => + !dbIntegrationRelations.some( + (dbRelation) => + dbRelation.itemId === inputRelation.itemId && + dbRelation.integrationId === inputRelation.integrationId, + ), + ); - if (sectionIds.length > 0) { - await transaction.delete(sections).where(inArray(sections.id, sectionIds)); - } + if (addedIntegrationRelations.length > 0) { + await transaction.insert(schema.integrationItems).values( + addedIntegrationRelations.map((relation) => ({ + itemId: relation.itemId, + integrationId: relation.integrationId, + })), + ); + } + + const updatedItems = filterUpdatedItems(inputItems, dbItems); + + for (const item of updatedItems) { + await transaction + .update(schema.items) + .set({ + kind: item.kind, + height: item.height, + width: item.width, + xOffset: item.xOffset, + yOffset: item.yOffset, + options: superjson.stringify(item.options), + advancedOptions: superjson.stringify(item.advancedOptions), + sectionId: item.sectionId, + }) + .where(eq(items.id, item.id)); + } + + const updatedSections = filterUpdatedItems(input.sections, dbBoard.sections); + + for (const section of updatedSections) { + const prev = dbBoard.sections.find((dbSection) => dbSection.id === section.id); + await transaction + .update(schema.sections) + .set({ + yOffset: section.yOffset, + xOffset: section.xOffset, + height: prev?.kind === "dynamic" && "height" in section ? section.height : null, + width: prev?.kind === "dynamic" && "width" in section ? section.width : null, + parentSectionId: + prev?.kind === "dynamic" && "parentSectionId" in section ? section.parentSectionId : null, + name: prev?.kind === "category" && "name" in section ? section.name : null, + }) + .where(eq(sections.id, section.id)); + } + + const removedIntegrationRelations = dbIntegrationRelations.filter( + (dbRelation) => + !inputIntegrationRelations.some( + (inputRelation) => + dbRelation.itemId === inputRelation.itemId && + dbRelation.integrationId === inputRelation.integrationId, + ), + ); + + for (const relation of removedIntegrationRelations) { + await transaction + .delete(schema.integrationItems) + .where( + and( + eq(integrationItems.itemId, relation.itemId), + eq(integrationItems.integrationId, relation.integrationId), + ), + ); + } + + const removedItems = filterRemovedItems(inputItems, dbItems); + + const itemIds = removedItems.map((item) => item.id); + if (itemIds.length > 0) { + await transaction.delete(schema.items).where(inArray(items.id, itemIds)); + } + + const removedSections = filterRemovedItems(input.sections, dbBoard.sections); + const sectionIds = removedSections.map((section) => section.id); + + if (sectionIds.length > 0) { + await transaction.delete(schema.sections).where(inArray(sections.id, sectionIds)); + } + }); + }, + handleSync(db) { + db.transaction((transaction) => { + const addedSections = filterAddedItems(input.sections, dbBoard.sections); + + if (addedSections.length > 0) { + transaction + .insert(sections) + .values( + addedSections.map((section) => ({ + id: section.id, + kind: section.kind, + yOffset: section.yOffset, + xOffset: section.kind === "dynamic" ? section.xOffset : 0, + height: "height" in section ? section.height : null, + width: "width" in section ? section.width : null, + parentSectionId: "parentSectionId" in section ? section.parentSectionId : null, + name: "name" in section ? section.name : null, + boardId: dbBoard.id, + })), + ) + .run(); + } + + const inputItems = input.sections.flatMap((section) => + section.items.map((item) => ({ ...item, sectionId: section.id })), + ); + const dbItems = dbBoard.sections.flatMap((section) => + section.items.map((item) => ({ ...item, sectionId: section.id })), + ); + + const addedItems = filterAddedItems(inputItems, dbItems); + + if (addedItems.length > 0) { + transaction + .insert(items) + .values( + addedItems.map((item) => ({ + id: item.id, + kind: item.kind, + height: item.height, + width: item.width, + xOffset: item.xOffset, + yOffset: item.yOffset, + options: superjson.stringify(item.options), + advancedOptions: superjson.stringify(item.advancedOptions), + sectionId: item.sectionId, + })), + ) + .run(); + } + + const inputIntegrationRelations = inputItems.flatMap(({ integrationIds, id: itemId }) => + integrationIds.map((integrationId) => ({ + integrationId, + itemId, + })), + ); + const dbIntegrationRelations = dbItems.flatMap(({ integrationIds, id: itemId }) => + integrationIds.map((integrationId) => ({ + integrationId, + itemId, + })), + ); + const addedIntegrationRelations = inputIntegrationRelations.filter( + (inputRelation) => + !dbIntegrationRelations.some( + (dbRelation) => + dbRelation.itemId === inputRelation.itemId && + dbRelation.integrationId === inputRelation.integrationId, + ), + ); + + if (addedIntegrationRelations.length > 0) { + transaction + .insert(integrationItems) + .values( + addedIntegrationRelations.map((relation) => ({ + itemId: relation.itemId, + integrationId: relation.integrationId, + })), + ) + .run(); + } + + const updatedItems = filterUpdatedItems(inputItems, dbItems); + + for (const item of updatedItems) { + transaction + .update(items) + .set({ + kind: item.kind, + height: item.height, + width: item.width, + xOffset: item.xOffset, + yOffset: item.yOffset, + options: superjson.stringify(item.options), + advancedOptions: superjson.stringify(item.advancedOptions), + sectionId: item.sectionId, + }) + .where(eq(items.id, item.id)) + .run(); + } + + const updatedSections = filterUpdatedItems(input.sections, dbBoard.sections); + + for (const section of updatedSections) { + const prev = dbBoard.sections.find((dbSection) => dbSection.id === section.id); + transaction + .update(sections) + .set({ + yOffset: section.yOffset, + xOffset: section.xOffset, + height: prev?.kind === "dynamic" && "height" in section ? section.height : null, + width: prev?.kind === "dynamic" && "width" in section ? section.width : null, + parentSectionId: + prev?.kind === "dynamic" && "parentSectionId" in section ? section.parentSectionId : null, + name: prev?.kind === "category" && "name" in section ? section.name : null, + }) + .where(eq(sections.id, section.id)) + .run(); + } + + const removedIntegrationRelations = dbIntegrationRelations.filter( + (dbRelation) => + !inputIntegrationRelations.some( + (inputRelation) => + dbRelation.itemId === inputRelation.itemId && + dbRelation.integrationId === inputRelation.integrationId, + ), + ); + + for (const relation of removedIntegrationRelations) { + transaction + .delete(integrationItems) + .where( + and( + eq(integrationItems.itemId, relation.itemId), + eq(integrationItems.integrationId, relation.integrationId), + ), + ) + .run(); + } + + const removedItems = filterRemovedItems(inputItems, dbItems); + + const itemIds = removedItems.map((item) => item.id); + if (itemIds.length > 0) { + transaction.delete(items).where(inArray(items.id, itemIds)).run(); + } + + const removedSections = filterRemovedItems(input.sections, dbBoard.sections); + const sectionIds = removedSections.map((section) => section.id); + + if (sectionIds.length > 0) { + transaction.delete(sections).where(inArray(sections.id, sectionIds)).run(); + } + }); + }, }); }), @@ -655,18 +874,42 @@ export const boardRouter = createTRPCRouter({ .mutation(async ({ input, ctx }) => { await throwIfActionForbiddenAsync(ctx, eq(boards.id, input.entityId), "full"); - await ctx.db.transaction(async (transaction) => { - await transaction.delete(boardUserPermissions).where(eq(boardUserPermissions.boardId, input.entityId)); - if (input.permissions.length === 0) { - return; - } - await transaction.insert(boardUserPermissions).values( - input.permissions.map((permission) => ({ - userId: permission.principalId, - permission: permission.permission, - boardId: input.entityId, - })), - ); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + await transaction + .delete(schema.boardUserPermissions) + .where(eq(boardUserPermissions.boardId, input.entityId)); + if (input.permissions.length === 0) { + return; + } + await transaction.insert(schema.boardUserPermissions).values( + input.permissions.map((permission) => ({ + userId: permission.principalId, + permission: permission.permission, + boardId: input.entityId, + })), + ); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction.delete(boardUserPermissions).where(eq(boardUserPermissions.boardId, input.entityId)).run(); + if (input.permissions.length === 0) { + return; + } + transaction + .insert(boardUserPermissions) + .values( + input.permissions.map((permission) => ({ + userId: permission.principalId, + permission: permission.permission, + boardId: input.entityId, + })), + ) + .run(); + }); + }, }); }), saveGroupBoardPermissions: protectedProcedure @@ -674,18 +917,42 @@ export const boardRouter = createTRPCRouter({ .mutation(async ({ input, ctx }) => { await throwIfActionForbiddenAsync(ctx, eq(boards.id, input.entityId), "full"); - await ctx.db.transaction(async (transaction) => { - await transaction.delete(boardGroupPermissions).where(eq(boardGroupPermissions.boardId, input.entityId)); - if (input.permissions.length === 0) { - return; - } - await transaction.insert(boardGroupPermissions).values( - input.permissions.map((permission) => ({ - groupId: permission.principalId, - permission: permission.permission, - boardId: input.entityId, - })), - ); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + await transaction + .delete(schema.boardGroupPermissions) + .where(eq(boardGroupPermissions.boardId, input.entityId)); + if (input.permissions.length === 0) { + return; + } + await transaction.insert(schema.boardGroupPermissions).values( + input.permissions.map((permission) => ({ + groupId: permission.principalId, + permission: permission.permission, + boardId: input.entityId, + })), + ); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction.delete(boardGroupPermissions).where(eq(boardGroupPermissions.boardId, input.entityId)).run(); + if (input.permissions.length === 0) { + return; + } + transaction + .insert(boardGroupPermissions) + .values( + input.permissions.map((permission) => ({ + groupId: permission.principalId, + permission: permission.permission, + boardId: input.entityId, + })), + ) + .run(); + }); + }, }); }), importOldmarrConfig: permissionRequiredProcedure diff --git a/packages/api/src/router/integration/integration-router.ts b/packages/api/src/router/integration/integration-router.ts index 7a1345557..9b309afcc 100644 --- a/packages/api/src/router/integration/integration-router.ts +++ b/packages/api/src/router/integration/integration-router.ts @@ -3,7 +3,7 @@ import { TRPCError } from "@trpc/server"; import { objectEntries } from "@homarr/common"; import { decryptSecret, encryptSecret } from "@homarr/common/server"; import type { Database } from "@homarr/db"; -import { and, asc, createId, eq, inArray, like } from "@homarr/db"; +import { and, asc, createId, eq, handleTransactionsAsync, inArray, like } from "@homarr/db"; import { groupMembers, groupPermissions, @@ -360,20 +360,45 @@ export const integrationRouter = createTRPCRouter({ .mutation(async ({ input, ctx }) => { await throwIfActionForbiddenAsync(ctx, eq(integrations.id, input.entityId), "full"); - await ctx.db.transaction(async (transaction) => { - await transaction - .delete(integrationUserPermissions) - .where(eq(integrationUserPermissions.integrationId, input.entityId)); - if (input.permissions.length === 0) { - return; - } - await transaction.insert(integrationUserPermissions).values( - input.permissions.map((permission) => ({ - userId: permission.principalId, - permission: permission.permission, - integrationId: input.entityId, - })), - ); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await ctx.db.transaction(async (transaction) => { + await transaction + .delete(schema.integrationUserPermissions) + .where(eq(schema.integrationUserPermissions.integrationId, input.entityId)); + if (input.permissions.length === 0) { + return; + } + await transaction.insert(schema.integrationUserPermissions).values( + input.permissions.map((permission) => ({ + userId: permission.principalId, + permission: permission.permission, + integrationId: input.entityId, + })), + ); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction + .delete(integrationUserPermissions) + .where(eq(integrationUserPermissions.integrationId, input.entityId)) + .run(); + if (input.permissions.length === 0) { + return; + } + transaction + .insert(integrationUserPermissions) + .values( + input.permissions.map((permission) => ({ + userId: permission.principalId, + permission: permission.permission, + integrationId: input.entityId, + })), + ) + .run(); + }); + }, }); }), saveGroupIntegrationPermissions: protectedProcedure @@ -381,20 +406,45 @@ export const integrationRouter = createTRPCRouter({ .mutation(async ({ input, ctx }) => { await throwIfActionForbiddenAsync(ctx, eq(integrations.id, input.entityId), "full"); - await ctx.db.transaction(async (transaction) => { - await transaction - .delete(integrationGroupPermissions) - .where(eq(integrationGroupPermissions.integrationId, input.entityId)); - if (input.permissions.length === 0) { - return; - } - await transaction.insert(integrationGroupPermissions).values( - input.permissions.map((permission) => ({ - groupId: permission.principalId, - permission: permission.permission, - integrationId: input.entityId, - })), - ); + await handleTransactionsAsync(ctx.db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + await transaction + .delete(schema.integrationGroupPermissions) + .where(eq(schema.integrationGroupPermissions.integrationId, input.entityId)); + if (input.permissions.length === 0) { + return; + } + await transaction.insert(schema.integrationGroupPermissions).values( + input.permissions.map((permission) => ({ + groupId: permission.principalId, + permission: permission.permission, + integrationId: input.entityId, + })), + ); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + transaction + .delete(integrationGroupPermissions) + .where(eq(integrationGroupPermissions.integrationId, input.entityId)) + .run(); + if (input.permissions.length === 0) { + return; + } + transaction + .insert(integrationGroupPermissions) + .values( + input.permissions.map((permission) => ({ + groupId: permission.principalId, + permission: permission.permission, + integrationId: input.entityId, + })), + ) + .run(); + }); + }, }); }), searchInIntegration: protectedProcedure diff --git a/packages/cron-jobs/src/jobs/icons-updater.ts b/packages/cron-jobs/src/jobs/icons-updater.ts index e818020d6..d278e7b87 100644 --- a/packages/cron-jobs/src/jobs/icons-updater.ts +++ b/packages/cron-jobs/src/jobs/icons-updater.ts @@ -1,7 +1,7 @@ import { splitToNChunks, Stopwatch } from "@homarr/common"; import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions"; import type { InferInsertModel } from "@homarr/db"; -import { db, inArray, sql } from "@homarr/db"; +import { db, handleTransactionsAsync, inArray, sql } from "@homarr/db"; import { createId } from "@homarr/db/client"; import { iconRepositories, icons } from "@homarr/db/schema"; import { fetchIconsAsync } from "@homarr/icons"; @@ -83,43 +83,81 @@ export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, { (iconRepository) => !repositoryIconGroups.some((group) => group.slug === iconRepository.slug), ); - db.transaction((transaction) => { - if (newIconRepositories.length >= 1) { - transaction.insert(iconRepositories).values(newIconRepositories).run(); - } + await handleTransactionsAsync(db, { + async handleAsync(db, schema) { + await db.transaction(async (transaction) => { + if (newIconRepositories.length >= 1) { + await transaction.insert(schema.iconRepositories).values(newIconRepositories); + } - if (newIcons.length >= 1) { - // We only insert 5000 icons at a time to avoid SQLite limitations - for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { - transaction.insert(icons).values(chunck).run(); - } - } - if (deadIcons.length >= 1) { - transaction - .delete(icons) - .where( - inArray( - // Combine iconRepositoryId and checksum to allow same icons on different repositories - sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`, - deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`), - ), - ) - .run(); - } + if (newIcons.length >= 1) { + // We only insert 5000 icons at a time to avoid SQLite limitations + for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { + await transaction.insert(schema.icons).values(chunck); + } + } + if (deadIcons.length >= 1) { + await transaction.delete(schema.icons).where( + inArray( + // Combine iconRepositoryId and checksum to allow same icons on different repositories + sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`, + deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`), + ), + ); + } - if (deadIconRepositories.length >= 1) { - transaction - .delete(iconRepositories) - .where( - inArray( - iconRepositories.id, - deadIconRepositories.map((iconRepository) => iconRepository.id), - ), - ) - .run(); - } + if (deadIconRepositories.length >= 1) { + await transaction.delete(schema.iconRepositories).where( + inArray( + iconRepositories.id, + deadIconRepositories.map((iconRepository) => iconRepository.id), + ), + ); + } - countDeleted += deadIcons.length; + countDeleted += deadIcons.length; + }); + }, + handleSync() { + db.transaction((transaction) => { + if (newIconRepositories.length >= 1) { + transaction.insert(iconRepositories).values(newIconRepositories).run(); + } + + if (newIcons.length >= 1) { + // We only insert 5000 icons at a time to avoid SQLite limitations + for (const chunck of splitToNChunks(newIcons, Math.ceil(newIcons.length / 5000))) { + transaction.insert(icons).values(chunck).run(); + } + } + if (deadIcons.length >= 1) { + transaction + .delete(icons) + .where( + inArray( + // Combine iconRepositoryId and checksum to allow same icons on different repositories + sql`concat(${icons.iconRepositoryId}, '.', ${icons.checksum})`, + deadIcons.map((icon) => `${icon.iconRepositoryId}.${icon.checksum}`), + ), + ) + .run(); + } + + if (deadIconRepositories.length >= 1) { + transaction + .delete(iconRepositories) + .where( + inArray( + iconRepositories.id, + deadIconRepositories.map((iconRepository) => iconRepository.id), + ), + ) + .run(); + } + + countDeleted += deadIcons.length; + }); + }, }); logger.info(`Updated database within ${stopWatch.getElapsedInHumanWords()} (-${countDeleted}, +${countInserted})`); diff --git a/packages/db/driver.ts b/packages/db/driver.ts index 643651d78..2dce7f4a0 100644 --- a/packages/db/driver.ts +++ b/packages/db/driver.ts @@ -2,6 +2,7 @@ import Database from "better-sqlite3"; import type { Logger } from "drizzle-orm"; import type { BetterSQLite3Database } from "drizzle-orm/better-sqlite3"; import { drizzle as drizzleSqlite } from "drizzle-orm/better-sqlite3"; +import type { MySql2Database } from "drizzle-orm/mysql2"; import { drizzle as drizzleMysql } from "drizzle-orm/mysql2"; import mysql from "mysql2"; @@ -11,7 +12,8 @@ import { env } from "./env"; import * as mysqlSchema from "./schema/mysql"; import * as sqliteSchema from "./schema/sqlite"; -type HomarrDatabase = BetterSQLite3Database; +export type HomarrDatabase = BetterSQLite3Database; +export type HomarrDatabaseMysql = MySql2Database; const init = () => { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition diff --git a/packages/db/index.ts b/packages/db/index.ts index 33575cb57..ac54017c0 100644 --- a/packages/db/index.ts +++ b/packages/db/index.ts @@ -7,5 +7,7 @@ export * from "drizzle-orm"; export const db = database; export type Database = typeof db; +export type { HomarrDatabaseMysql } from "./driver"; export { createId } from "@paralleldrive/cuid2"; +export { handleDiffrentDbDriverOperationsAsync as handleTransactionsAsync } from "./transactions"; diff --git a/packages/db/transactions.ts b/packages/db/transactions.ts new file mode 100644 index 000000000..e7da4ae16 --- /dev/null +++ b/packages/db/transactions.ts @@ -0,0 +1,24 @@ +import type { HomarrDatabase, HomarrDatabaseMysql } from "./driver"; +import { env } from "./env"; +import * as mysqlSchema from "./schema/mysql"; + +type MysqlSchema = typeof mysqlSchema; + +interface HandleTransactionInput { + handleAsync: (db: HomarrDatabaseMysql, schema: MysqlSchema) => Promise; + handleSync: (db: HomarrDatabase) => void; +} + +/** + * The below method is mostly used to handle transactions in different database drivers. + * As better-sqlite3 transactions have to be synchronous, we have to implement them in a different way. + * But it can also generally be used when dealing with different database drivers. + */ +export const handleDiffrentDbDriverOperationsAsync = async (db: HomarrDatabase, input: HandleTransactionInput) => { + if (env.DB_DRIVER !== "mysql2") { + input.handleSync(db); + return; + } + + await input.handleAsync(db as unknown as HomarrDatabaseMysql, mysqlSchema); +}; diff --git a/packages/old-import/src/import/collections/common.ts b/packages/old-import/src/import/collections/common.ts index 4065063b5..a7ad01da4 100644 --- a/packages/old-import/src/import/collections/common.ts +++ b/packages/old-import/src/import/collections/common.ts @@ -1,5 +1,5 @@ import { objectEntries } from "@homarr/common"; -import type { Database, InferInsertModel } from "@homarr/db"; +import type { Database, HomarrDatabaseMysql, InferInsertModel } from "@homarr/db"; import * as schema from "@homarr/db/schema"; type TableKey = { @@ -29,5 +29,15 @@ export const createDbInsertCollection = (tablesInIns } }); }, + insertAllAsync: async (db: HomarrDatabaseMysql) => { + await db.transaction(async (transaction) => { + for (const [key, values] of objectEntries(context)) { + if (values.length >= 1) { + // Below is actually the mysqlSchema when the driver is mysql + await transaction.insert(schema[key] as never).values(values as never); + } + } + }); + }, }; }; diff --git a/packages/old-import/src/import/import-initial-oldmarr.ts b/packages/old-import/src/import/import-initial-oldmarr.ts index 90b2fea2a..fdf1a79da 100644 --- a/packages/old-import/src/import/import-initial-oldmarr.ts +++ b/packages/old-import/src/import/import-initial-oldmarr.ts @@ -1,6 +1,7 @@ import type { z } from "zod"; import { Stopwatch } from "@homarr/common"; +import { handleTransactionsAsync } from "@homarr/db"; import type { Database } from "@homarr/db"; import { logger } from "@homarr/log"; @@ -34,11 +35,21 @@ export const importInitialOldmarrAsync = async ( logger.info("Inserting import data to database"); - // Due to a limitation with better-sqlite it's only possible to use it synchronously - db.transaction((transaction) => { - boardInsertCollection.insertAll(transaction); - userInsertCollection.insertAll(transaction); - integrationInsertCollection.insertAll(transaction); + await handleTransactionsAsync(db, { + async handleAsync(db) { + await db.transaction(async (transaction) => { + await boardInsertCollection.insertAllAsync(transaction); + await userInsertCollection.insertAllAsync(transaction); + await integrationInsertCollection.insertAllAsync(transaction); + }); + }, + handleSync(db) { + db.transaction((transaction) => { + boardInsertCollection.insertAll(transaction); + userInsertCollection.insertAll(transaction); + integrationInsertCollection.insertAll(transaction); + }); + }, }); logger.info(`Import successful (in ${stopwatch.getElapsedInHumanWords()})`); diff --git a/packages/old-import/src/import/import-single-oldmarr.ts b/packages/old-import/src/import/import-single-oldmarr.ts index 42bcc5c15..2ca6ed4c1 100644 --- a/packages/old-import/src/import/import-single-oldmarr.ts +++ b/packages/old-import/src/import/import-single-oldmarr.ts @@ -1,4 +1,4 @@ -import { inArray } from "@homarr/db"; +import { handleTransactionsAsync, inArray } from "@homarr/db"; import type { Database } from "@homarr/db"; import { apps } from "@homarr/db/schema"; import type { OldmarrConfig } from "@homarr/old-schema"; @@ -31,6 +31,12 @@ export const importSingleOldmarrConfigAsync = async ( const boardInsertCollection = createBoardInsertCollection({ preparedApps, preparedBoards }, settings); - // Due to a limitation with better-sqlite it's only possible to use it synchronously - boardInsertCollection.insertAll(db); + await handleTransactionsAsync(db, { + async handleAsync(db) { + await boardInsertCollection.insertAllAsync(db); + }, + handleSync(db) { + boardInsertCollection.insertAll(db); + }, + }); };