refactor: add cron job core package (#704)

* refactor: add cron job core package

* docs: add comments to cron validation types

* chore(deps): move node-cron dependencies from tasks app to cron-jobs-core package

* fix: runOnInit is not running on start and rather on job creation

* fix: format issues

* fix: build fails when using top level await in cjs

* chore: update turbo gen package json typescript version to 5.5.2

* fix: format issue

* fix: deepsource issues

* chore: update turbo gen package json eslint version to 9.5.0

* chore: fix frozen lockfile and format
This commit is contained in:
Meier Lukas
2024-06-22 20:00:20 +02:00
committed by GitHub
parent ea12da991a
commit 92afd82d22
26 changed files with 397 additions and 128 deletions

View File

@@ -31,8 +31,8 @@
"@homarr/widgets": "workspace:^0.1.0",
"@homarr/validation": "workspace:^0.1.0",
"@homarr/analytics": "workspace:^0.1.0",
"@homarr/cron-jobs-core": "workspace:^0.1.0",
"dotenv": "^16.4.5",
"node-cron": "^3.0.3",
"superjson": "2.2.1",
"undici": "6.19.2"
},
@@ -40,7 +40,6 @@
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"@types/node-cron": "^3.0.11",
"@types/node": "^20.14.8",
"dotenv-cli": "^7.4.2",
"eslint": "^9.5.0",

View File

@@ -3,9 +3,9 @@ import { smartHomeEntityStateJob } from "~/jobs/integrations/home-assistant";
import { analyticsJob } from "./jobs/analytics";
import { pingJob } from "./jobs/ping";
import { queuesJob } from "./jobs/queue";
import { createJobGroup } from "./lib/cron-job/group";
import { createCronJobGroup } from "./lib/jobs";
export const jobs = createJobGroup({
export const jobs = createCronJobGroup({
// Add your jobs here:
analytics: analyticsJob,
iconsUpdater: iconsUpdaterJob,

View File

@@ -1,14 +1,14 @@
import SuperJSON from "superjson";
import { sendServerAnalyticsAsync } from "@homarr/analytics";
import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions";
import { db, eq } from "@homarr/db";
import { serverSettings } from "@homarr/db/schema/sqlite";
import { EVERY_WEEK } from "~/lib/cron-job/constants";
import { createCronJob } from "~/lib/cron-job/creator";
import { createCronJob } from "~/lib/jobs";
import type { defaultServerSettings } from "../../../../packages/server-settings";
export const analyticsJob = createCronJob(EVERY_WEEK, {
export const analyticsJob = createCronJob("analytics", EVERY_WEEK, {
runOnStart: true,
}).withCallback(async () => {
const analyticSetting = await db.query.serverSettings.findFirst({

View File

@@ -1,4 +1,5 @@
import { Stopwatch } from "@homarr/common";
import { EVERY_WEEK } from "@homarr/cron-jobs-core/expressions";
import type { InferInsertModel } from "@homarr/db";
import { db, inArray } from "@homarr/db";
import { createId } from "@homarr/db/client";
@@ -6,10 +7,9 @@ import { iconRepositories, icons } from "@homarr/db/schema/sqlite";
import { fetchIconsAsync } from "@homarr/icons";
import { logger } from "@homarr/log";
import { EVERY_WEEK } from "~/lib/cron-job/constants";
import { createCronJob } from "~/lib/cron-job/creator";
import { createCronJob } from "~/lib/jobs";
export const iconsUpdaterJob = createCronJob(EVERY_WEEK, {
export const iconsUpdaterJob = createCronJob("iconsUpdater", EVERY_WEEK, {
runOnStart: true,
}).withCallback(async () => {
logger.info("Updating icon repository cache...");

View File

@@ -1,6 +1,7 @@
import SuperJSON from "superjson";
import { decryptSecret } from "@homarr/common";
import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
import { db, eq } from "@homarr/db";
import { items } from "@homarr/db/schema/sqlite";
import { HomeAssistantIntegration } from "@homarr/integrations";
@@ -8,10 +9,9 @@ import { logger } from "@homarr/log";
import { homeAssistantEntityState } from "@homarr/redis";
import type { WidgetComponentProps } from "@homarr/widgets";
import { EVERY_MINUTE } from "~/lib/cron-job/constants";
import { createCronJob } from "~/lib/cron-job/creator";
import { createCronJob } from "~/lib/jobs";
export const smartHomeEntityStateJob = createCronJob(EVERY_MINUTE).withCallback(async () => {
export const smartHomeEntityStateJob = createCronJob("smartHomeEntityState", EVERY_MINUTE).withCallback(async () => {
const itemsForIntegration = await db.query.items.findMany({
where: eq(items.kind, "smartHome-entityState"),
with: {

View File

@@ -1,11 +1,11 @@
import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
import { logger } from "@homarr/log";
import { sendPingRequestAsync } from "@homarr/ping";
import { pingChannel, pingUrlChannel } from "@homarr/redis";
import { EVERY_MINUTE } from "~/lib/cron-job/constants";
import { createCronJob } from "~/lib/cron-job/creator";
import { createCronJob } from "~/lib/jobs";
export const pingJob = createCronJob(EVERY_MINUTE).withCallback(async () => {
export const pingJob = createCronJob("ping", EVERY_MINUTE).withCallback(async () => {
const urls = await pingUrlChannel.getAllAsync();
for (const url of new Set(urls)) {

View File

@@ -1,8 +1,9 @@
import { EVERY_MINUTE } from "../lib/cron-job/constants";
import { createCronJob } from "../lib/cron-job/creator";
import { EVERY_MINUTE } from "@homarr/cron-jobs-core/expressions";
import { createCronJob } from "~/lib/jobs";
import { queueWorkerAsync } from "../lib/queue/worker";
// This job processes queues, it runs every minute.
export const queuesJob = createCronJob(EVERY_MINUTE).withCallback(async () => {
export const queuesJob = createCronJob("queues", EVERY_MINUTE).withCallback(async () => {
await queueWorkerAsync();
});

View File

@@ -1,7 +0,0 @@
export const EVERY_5_SECONDS = "*/5 * * * * *";
export const EVERY_MINUTE = "* * * * *";
export const EVERY_5_MINUTES = "*/5 * * * *";
export const EVERY_10_MINUTES = "*/10 * * * *";
export const EVERY_HOUR = "0 * * * *";
export const EVERY_DAY = "0 0 * * */1";
export const EVERY_WEEK = "0 0 * * 1";

View File

@@ -1,37 +0,0 @@
import cron from "node-cron";
import type { MaybePromise } from "@homarr/common/types";
import { logger } from "@homarr/log";
interface CreateCronJobOptions {
runOnStart?: boolean;
}
export const createCronJob = (cronExpression: string, options: CreateCronJobOptions = { runOnStart: false }) => {
return {
withCallback: (callback: () => MaybePromise<void>) => {
const catchingCallbackAsync = async () => {
try {
await callback();
} catch (error) {
logger.error(
`apps/tasks/src/lib/cron-job/creator.ts: The callback of a cron job failed, expression ${cronExpression}, with error:`,
error,
);
}
};
if (options.runOnStart) {
void catchingCallbackAsync();
}
const task = cron.schedule(cronExpression, () => void catchingCallbackAsync(), {
scheduled: false,
});
return {
_expression: cronExpression,
_task: task,
};
},
};
};

View File

@@ -1,45 +0,0 @@
import { objectEntries } from "@homarr/common";
import type { createCronJob } from "./creator";
import { jobRegistry } from "./registry";
type Jobs = Record<string, ReturnType<ReturnType<typeof createCronJob>["withCallback"]>>;
export const createJobGroup = <TJobs extends Jobs>(jobs: TJobs) => {
for (const [name, job] of objectEntries(jobs)) {
if (typeof name !== "string") continue;
jobRegistry.set(name, {
name,
expression: job._expression,
active: false,
task: job._task,
});
}
return {
start: (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
job.active = true;
job.task.start();
},
startAll: () => {
for (const job of jobRegistry.values()) {
job.active = true;
job.task.start();
}
},
stop: (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
job.active = false;
job.task.stop();
},
stopAll: () => {
for (const job of jobRegistry.values()) {
job.active = false;
job.task.stop();
}
},
};
};

View File

@@ -1,9 +0,0 @@
import type cron from "node-cron";
interface Job {
name: string;
expression: string;
active: boolean;
task: cron.ScheduledTask;
}
export const jobRegistry = new Map<string, Job>();

View File

@@ -0,0 +1,21 @@
import { createCronJobFunctions } from "@homarr/cron-jobs-core";
import type { Logger } from "@homarr/cron-jobs-core/logger";
import { logger } from "@homarr/log";
class WinstonCronJobLogger implements Logger {
logDebug(message: string) {
logger.debug(message);
}
logInfo(message: string) {
logger.info(message);
}
logError(error: unknown) {
logger.error(error);
}
}
export const { createCronJob, createCronJobGroup } = createCronJobFunctions({
logger: new WinstonCronJobLogger(),
});

View File

@@ -4,6 +4,7 @@ import "./undici-log-agent-override";
import { jobs } from "./jobs";
import { seedServerSettingsAsync } from "./seed-server-settings";
jobs.startAll();
void seedServerSettingsAsync();
void (async () => {
await jobs.startAllAsync();
await seedServerSettingsAsync();
})();

View File

@@ -0,0 +1,9 @@
import baseConfig from "@homarr/eslint-config/base";
/** @type {import('typescript-eslint').Config} */
export default [
{
ignores: [],
},
...baseConfig,
];

View File

@@ -0,0 +1 @@
export * from "./src";

View File

@@ -0,0 +1,38 @@
{
"name": "@homarr/cron-jobs-core",
"private": true,
"version": "0.1.0",
"type": "module",
"exports": {
".": "./index.ts",
"./expressions": "./src/expressions.ts",
"./logger": "./src/logger.ts"
},
"typesVersions": {
"*": {
"*": [
"src/*"
]
}
},
"license": "MIT",
"scripts": {
"clean": "rm -rf .turbo node_modules",
"lint": "eslint",
"format": "prettier --check . --ignore-path ../../.gitignore",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"node-cron": "^3.0.3",
"@homarr/common": "workspace:^0.1.0"
},
"devDependencies": {
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"@types/node-cron": "^3.0.11",
"eslint": "^9.5.0",
"typescript": "^5.5.2"
},
"prettier": "@homarr/prettier-config"
}

View File

@@ -0,0 +1,94 @@
import cron from "node-cron";
import type { MaybePromise } from "@homarr/common/types";
import type { Logger } from "./logger";
import type { ValidateCron } from "./validation";
export interface CreateCronJobCreatorOptions<TAllowedNames extends string> {
beforeCallback?: (name: TAllowedNames) => MaybePromise<void>;
onCallbackSuccess?: (name: TAllowedNames) => MaybePromise<void>;
onCallbackError?: (name: TAllowedNames, error: unknown) => MaybePromise<void>;
timezone?: string;
logger: Logger;
}
interface CreateCronJobOptions {
runOnStart?: boolean;
}
const createCallback = <TAllowedNames extends string, TName extends TAllowedNames>(
name: TName,
cronExpression: string,
options: CreateCronJobOptions,
creatorOptions: CreateCronJobCreatorOptions<TAllowedNames>,
) => {
return (callback: () => MaybePromise<void>) => {
const catchingCallbackAsync = async () => {
try {
creatorOptions.logger.logDebug(`The callback of '${name}' cron job started`);
await creatorOptions.beforeCallback?.(name);
await callback();
creatorOptions.logger.logInfo(`The callback of '${name}' cron job succeeded`);
await creatorOptions.onCallbackSuccess?.(name);
} catch (error) {
creatorOptions.logger.logError(error);
await creatorOptions.onCallbackError?.(name, error);
}
};
/**
* We are not using the runOnInit method as we want to run the job only once we start the cron job schedule manually.
* This allows us to always run it once we start it. Additionally it will not run the callback if only the cron job file is imported.
*/
const scheduledTask = cron.schedule(cronExpression, () => void catchingCallbackAsync(), {
scheduled: false,
name,
timezone: creatorOptions.timezone,
});
creatorOptions.logger.logDebug(
`The cron job '${name}' was created with expression ${cronExpression} in timezone ${creatorOptions.timezone} and runOnStart ${options.runOnStart}`,
);
return {
name,
cronExpression,
scheduledTask,
async onStartAsync() {
if (!options.runOnStart) return;
creatorOptions.logger.logDebug(`The cron job '${name}' is running because runOnStart is set to true`);
await catchingCallbackAsync();
},
};
};
};
export type JobCallback<TAllowedNames extends string, TName extends TAllowedNames> = ReturnType<
typeof createCallback<TAllowedNames, TName>
>;
export const createCronJobCreator = <TAllowedNames extends string = string>(
creatorOptions: CreateCronJobCreatorOptions<TAllowedNames>,
) => {
return <TName extends TAllowedNames, TExpression extends string>(
name: TName,
cronExpression: TExpression,
options: CreateCronJobOptions = { runOnStart: false },
) => {
creatorOptions.logger.logDebug(`Validating cron expression '${cronExpression}' for job: ${name}`);
if (!cron.validate(cronExpression)) {
throw new Error(`Invalid cron expression '${cronExpression}' for job '${name}'`);
}
creatorOptions.logger.logDebug(`Cron job expression '${cronExpression}' for job ${name} is valid`);
const returnValue = {
withCallback: createCallback<TAllowedNames, TName>(name, cronExpression, options, creatorOptions),
};
// This is a type guard to check if the cron expression is valid and give the user a type hint
return returnValue as unknown as ValidateCron<TExpression> extends true
? typeof returnValue
: "Invalid cron expression";
};
};

View File

@@ -0,0 +1,9 @@
import { checkCron } from "./validation";
export const EVERY_5_SECONDS = checkCron("*/5 * * * * *") satisfies string;
export const EVERY_MINUTE = checkCron("* * * * *") satisfies string;
export const EVERY_5_MINUTES = checkCron("*/5 * * * *") satisfies string;
export const EVERY_10_MINUTES = checkCron("*/10 * * * *") satisfies string;
export const EVERY_HOUR = checkCron("0 * * * *") satisfies string;
export const EVERY_DAY = checkCron("0 0 * * */1") satisfies string;
export const EVERY_WEEK = checkCron("0 0 * * 1") satisfies string;

View File

@@ -0,0 +1,64 @@
import { objectEntries } from "@homarr/common";
import type { JobCallback } from "./creator";
import type { Logger } from "./logger";
import { jobRegistry } from "./registry";
type Jobs<TAllowedNames extends string> = {
[name in TAllowedNames]: ReturnType<JobCallback<TAllowedNames, name>>;
};
export interface CreateCronJobGroupCreatorOptions {
logger: Logger;
}
export const createJobGroupCreator = <TAllowedNames extends string = string>(
options: CreateCronJobGroupCreatorOptions,
) => {
return <TJobs extends Jobs<TAllowedNames>>(jobs: TJobs) => {
options.logger.logDebug(`Creating job group with ${Object.keys(jobs).length} jobs.`);
for (const [key, job] of objectEntries(jobs)) {
if (typeof key !== "string" || typeof job.name !== "string") continue;
options.logger.logDebug(`Added job ${job.name} to the job registry.`);
jobRegistry.set(key, {
...job,
name: job.name,
});
}
return {
startAsync: async (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
options.logger.logInfo(`Starting schedule cron job ${job.name}.`);
await job.onStartAsync();
job.scheduledTask.start();
},
startAllAsync: async () => {
for (const job of jobRegistry.values()) {
options.logger.logInfo(`Starting schedule of cron job ${job.name}.`);
await job.onStartAsync();
job.scheduledTask.start();
}
},
stop: (name: keyof TJobs) => {
const job = jobRegistry.get(name as string);
if (!job) return;
options.logger.logInfo(`Stopping schedule cron job ${job.name}.`);
job.scheduledTask.stop();
},
stopAll: () => {
for (const job of jobRegistry.values()) {
options.logger.logInfo(`Stopping schedule cron job ${job.name}.`);
job.scheduledTask.stop();
}
},
getJobRegistry() {
return jobRegistry as Map<TAllowedNames, ReturnType<JobCallback<TAllowedNames, TAllowedNames>>>;
},
};
};
};

View File

@@ -0,0 +1,15 @@
import type { CreateCronJobCreatorOptions } from "./creator";
import { createCronJobCreator } from "./creator";
import { createJobGroupCreator } from "./group";
import { ConsoleLogger } from "./logger";
export const createCronJobFunctions = <TAllowedNames extends string>(
options: CreateCronJobCreatorOptions<TAllowedNames> = { logger: new ConsoleLogger() },
) => {
return {
createCronJob: createCronJobCreator<TAllowedNames>(options),
createCronJobGroup: createJobGroupCreator<TAllowedNames>({
logger: options.logger,
}),
};
};

View File

@@ -0,0 +1,19 @@
export interface Logger {
logDebug(message: string): void;
logInfo(message: string): void;
logError(error: unknown): void;
}
export class ConsoleLogger implements Logger {
public logDebug(message: string) {
console.log(message);
}
public logInfo(message: string) {
console.log(message);
}
public logError(error: unknown) {
console.error(error);
}
}

View File

@@ -0,0 +1,3 @@
import type { JobCallback } from "./creator";
export const jobRegistry = new Map<string, ReturnType<JobCallback<string, string>>>();

View File

@@ -0,0 +1,60 @@
// The below two types are used for a number with arbitrary length. By default the type `${number}` allows spaces which is not allowed in cron expressions.
type Digit = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;
type NumberWithoutSpaces = `${Digit}${number | ""}` & `${number | ""}${Digit}`;
// The below type is used to constrain the cron expression to only allow valid characters. This will find any invalid characters in the cron expression.
type CronChars = `${"*" | "/" | "-" | "," | NumberWithoutSpaces}`;
/**
* Will return false if the TMaybeCron string contains any invalid characters.
* Otherwise it will return true.
*/
type ConstrainedCronString<TMaybeCron extends string> = TMaybeCron extends ""
? true
: TMaybeCron extends `${CronChars}${infer Rest}`
? ConstrainedCronString<Rest>
: false;
/**
* Will return true if the TMaybeCron string is a valid cron expression.
* Otherwise it will return false.
*
* It allows cron expressions with 5 or 6 parts. (Seconds are optional)
* See https://nodecron.com/docs/
*/
export type ValidateCron<TMaybeCron extends string> =
TMaybeCron extends `${infer inferedSecond} ${infer inferedMinute} ${infer inferedHour} ${infer inferedMonthDay} ${infer inferedMonth} ${infer inferedWeekDay}`
? ConstrainedCronString<inferedSecond> extends true
? ConstrainedCronString<inferedMinute> extends true
? ConstrainedCronString<inferedHour> extends true
? ConstrainedCronString<inferedMonthDay> extends true
? ConstrainedCronString<inferedMonth> extends true
? ConstrainedCronString<inferedWeekDay> extends true
? true
: false
: false
: false
: false
: false
: false
: TMaybeCron extends `${infer inferedMinute} ${infer inferedHour} ${infer inferedMonthDay} ${infer inferedMonth} ${infer inferedWeekDay}`
? ConstrainedCronString<inferedMinute> extends true
? ConstrainedCronString<inferedHour> extends true
? ConstrainedCronString<inferedMonthDay> extends true
? ConstrainedCronString<inferedMonth> extends true
? ConstrainedCronString<inferedWeekDay> extends true
? true
: false
: false
: false
: false
: false
: false;
/**
* Will return the cron expression if it is valid.
* Otherwise it will return void (as type).
*/
export const checkCron = <const T extends string>(cron: T) => {
return cron as ValidateCron<T> extends true ? T : void;
};

View File

@@ -0,0 +1,8 @@
{
"extends": "@homarr/tsconfig/base.json",
"compilerOptions": {
"tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
},
"include": ["*.ts", "src"],
"exclude": ["node_modules"]
}

37
pnpm-lock.yaml generated
View File

@@ -258,6 +258,9 @@ importers:
'@homarr/common':
specifier: workspace:^0.1.0
version: link:../../packages/common
'@homarr/cron-jobs-core':
specifier: workspace:^0.1.0
version: link:../../packages/cron-jobs-core
'@homarr/db':
specifier: workspace:^0.1.0
version: link:../../packages/db
@@ -291,9 +294,6 @@ importers:
dotenv:
specifier: ^16.4.5
version: 16.4.5
node-cron:
specifier: ^3.0.3
version: 3.0.3
superjson:
specifier: 2.2.1
version: 2.2.1
@@ -313,9 +313,6 @@ importers:
'@types/node':
specifier: ^20.14.8
version: 20.14.8
'@types/node-cron':
specifier: ^3.0.11
version: 3.0.11
dotenv-cli:
specifier: ^7.4.2
version: 7.4.2
@@ -592,6 +589,34 @@ importers:
specifier: ^5.5.2
version: 5.5.2
packages/cron-jobs-core:
dependencies:
'@homarr/common':
specifier: workspace:^0.1.0
version: link:../common
node-cron:
specifier: ^3.0.3
version: 3.0.3
devDependencies:
'@homarr/eslint-config':
specifier: workspace:^0.2.0
version: link:../../tooling/eslint
'@homarr/prettier-config':
specifier: workspace:^0.1.0
version: link:../../tooling/prettier
'@homarr/tsconfig':
specifier: workspace:^0.1.0
version: link:../../tooling/typescript
'@types/node-cron':
specifier: ^3.0.11
version: 3.0.11
eslint:
specifier: ^9.5.0
version: 9.5.0
typescript:
specifier: ^5.5.2
version: 5.5.2
packages/db:
dependencies:
'@auth/core':

View File

@@ -24,8 +24,8 @@
"@homarr/eslint-config": "workspace:^0.2.0",
"@homarr/prettier-config": "workspace:^0.1.0",
"@homarr/tsconfig": "workspace:^0.1.0",
"eslint": "^9.4.0",
"typescript": "^5.4.5"
"eslint": "^9.5.0",
"typescript": "^5.5.2"
},
"prettier": "@homarr/prettier-config"
}