diff --git a/.changeset/pretty-tomatoes-shop.md b/.changeset/pretty-tomatoes-shop.md new file mode 100644 index 0000000..097951c --- /dev/null +++ b/.changeset/pretty-tomatoes-shop.md @@ -0,0 +1,5 @@ +--- +"@datatruck/cli": minor +--- + +Add cron server diff --git a/packages/cli/package.json b/packages/cli/package.json index cdfa464..7ccf49b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -12,6 +12,7 @@ "async": "^3.2.4", "chalk": "^4.1.2", "commander": "^11.0.0", + "croner": "^7.0.4", "dayjs": "^1.11.10", "fast-folder-size": "^2.2.0", "fast-glob": "^3.3.1", diff --git a/packages/cli/src/Command/StartServerCommand.ts b/packages/cli/src/Command/StartServerCommand.ts index 658cc63..357e39f 100644 --- a/packages/cli/src/Command/StartServerCommand.ts +++ b/packages/cli/src/Command/StartServerCommand.ts @@ -1,4 +1,5 @@ import { ConfigAction } from "../Action/ConfigAction"; +import { createCronServer } from "../utils/datatruck/cron-server"; import { createDatatruckRepositoryServer } from "../utils/datatruck/repository-server"; import { CommandAbstract } from "./CommandAbstract"; @@ -13,11 +14,14 @@ export class StartServerCommand extends CommandAbstract< } override async onExec() { const config = await ConfigAction.fromGlobalOptions(this.globalOptions); + const verbose = !!this.globalOptions.verbose; const log = config.server?.log ?? true; const repositoryOptions = config.server?.repository || {}; if (repositoryOptions.enabled ?? true) { - const server = createDatatruckRepositoryServer(repositoryOptions, log); + const server = createDatatruckRepositoryServer(repositoryOptions, { + log, + }); const port = repositoryOptions.listen?.port ?? 8888; const address = repositoryOptions.listen?.address ?? "127.0.0.1"; console.info( @@ -29,7 +33,20 @@ export class StartServerCommand extends CommandAbstract< }); server.listen(port, address); } + const cronOptions = config.server?.cron || {}; + if (cronOptions.enabled ?? true) { + if (typeof this.configPath !== "string") + throw new Error(`Config path is required by cron server`); + const server = createCronServer(cronOptions, { + verbose, + log, + configPath: this.configPath, + }); + server.start(); + console.info(`Cron server started`); + } + await new Promise(() => setInterval(() => {}, 60_000)); return 0; } } diff --git a/packages/cli/src/Config/Config.ts b/packages/cli/src/Config/Config.ts index 0f37112..6f54cc9 100644 --- a/packages/cli/src/Config/Config.ts +++ b/packages/cli/src/Config/Config.ts @@ -1,7 +1,11 @@ +import { backupCommandOptionDef } from "../Command/BackupCommand"; +import { copyCommandOptionsDef } from "../Command/CopyCommand"; import { DefinitionEnum, makeRef } from "../JsonSchema/DefinitionEnum"; import { ScriptTaskDefinitionEnum } from "../Task/ScriptTask"; import { FormatType, dataFormats } from "../utils/DataFormat"; -import { DatatruckServerOptions } from "../utils/datatruck/repository-server"; +import { DatatruckCronServerOptions } from "../utils/datatruck/cron-server"; +import { DatatruckRepositoryServerOptions } from "../utils/datatruck/repository-server"; +import { createCaseSchema, omitPropertySchema } from "../utils/schema"; import { Step } from "../utils/steps"; import { PackageConfigType } from "./PackageConfig"; import { PrunePolicyConfigType } from "./PrunePolicyConfig"; @@ -18,6 +22,12 @@ export type ConfigType = { prunePolicy?: PrunePolicyConfigType; }; +export type DatatruckServerOptions = { + log?: boolean; + repository?: DatatruckRepositoryServerOptions; + cron?: DatatruckCronServerOptions; +}; + export type ReportConfig = { when?: "success" | "error"; format?: Exclude; @@ -123,6 +133,42 @@ export const configDefinition: JSONSchema7 = { }, }, }, + cron: { + type: "object", + additionalProperties: false, + properties: { + enabled: { type: "boolean" }, + actions: { + type: "array", + items: { + allOf: [ + { + type: "object", + required: ["schedule"], + properties: { + schedule: { type: "string" }, + }, + }, + { + anyOf: createCaseSchema( + { + type: "type", + value: "options", + }, + { + backup: omitPropertySchema( + backupCommandOptionDef, + "dryRun", + ), + copy: copyCommandOptionsDef, + }, + ), + }, + ], + }, + }, + }, + }, }, }, prunePolicy: makeRef(DefinitionEnum.prunePolicy), diff --git a/packages/cli/src/utils/cli.ts b/packages/cli/src/utils/cli.ts index 587b3b8..7fbfc30 100644 --- a/packages/cli/src/utils/cli.ts +++ b/packages/cli/src/utils/cli.ts @@ -127,6 +127,28 @@ export function parseOptions( return result; } +export function stringifyOptions( + options: OptionsType, + object: any, +) { + const result: string[] = []; + for (const key in options) { + const fullOpt = options[key].option; + const [opt] = fullOpt.split(","); + const isNegative = fullOpt.startsWith("--no"); + const isBool = !fullOpt.includes("<") && !fullOpt.includes("["); + const defaultsValue = isNegative ? true : options[key].defaults; + const value = object?.[key] ?? defaultsValue; + + if (isBool) { + if (object[key]) result.push(opt); + } else if (value !== undefined) { + result.push(opt, `${value}`); + } + } + return result; +} + export function confirm(message: string) { const rl = createInterface({ input: process.stdin, diff --git a/packages/cli/src/utils/datatruck/cron-server.ts b/packages/cli/src/utils/datatruck/cron-server.ts new file mode 100644 index 0000000..ee9ec78 --- /dev/null +++ b/packages/cli/src/utils/datatruck/cron-server.ts @@ -0,0 +1,97 @@ +import { BackupCommandOptions } from "../../Command/BackupCommand"; +import { CopyCommandOptionsType } from "../../Command/CopyCommand"; +import { CommandConstructorFactory } from "../../Factory/CommandFactory"; +import { stringifyOptions } from "../cli"; +import { exec } from "../process"; +import { Cron } from "croner"; + +export type CronAction = + | { + schedule: string; + name: "backup"; + options: BackupCommandOptions; + } + | { + schedule: string; + name: "copy"; + options: CopyCommandOptionsType; + }; + +export type DatatruckCronServerOptions = { + enabled?: boolean; + actions?: CronAction[]; +}; + +function createJobs( + actions: CronAction[], + currentJobs: Cron[] = [], + worker: (action: CronAction, index: number) => Promise, +) { + const jobs: Cron[] = []; + for (const action of actions) { + const index = actions.indexOf(action); + const context = JSON.stringify({ + index: actions.indexOf(action), + data: action, + }); + const job = currentJobs.at(index); + if (!job || job.options.context !== context) { + job?.stop(); + jobs.push( + Cron( + action.schedule, + { + paused: true, + context: JSON.stringify(action), + catch: true, + protect: true, + }, + () => worker(action, index), + ), + ); + } + } + return jobs; +} + +export function createCronServer( + options: DatatruckCronServerOptions, + config: { + log: boolean; + verbose: boolean; + configPath: string; + }, +) { + const worker = async (action: CronAction, index: number) => { + if (config.log) console.info(`> [job] ${index} - ${action.name}`); + try { + const Command = CommandConstructorFactory(action.name as any); + const command = new Command( + { config: { packages: [], repositories: [] } }, + {}, + ); + const cliOptions = stringifyOptions(command.onOptions(), action.options); + const [node, bin] = process.argv; + await exec( + node, + [bin, "-c", config.configPath, action.name, ...cliOptions], + {}, + { log: config.verbose }, + ); + if (config.log) console.info(`< [job] ${index} - ${action.name}`); + } catch (error) { + if (config.log) console.error(`< [job] ${index} - ${action.name}`, error); + } + }; + + const jobs = createJobs(options.actions || [], [], worker); + + return { + start: () => { + for (const job of jobs) job.resume(); + }, + stop: () => { + for (const job of jobs) job.stop(); + }, + }; +} diff --git a/packages/cli/src/utils/datatruck/repository-server.ts b/packages/cli/src/utils/datatruck/repository-server.ts index 78989d5..2e37616 100644 --- a/packages/cli/src/utils/datatruck/repository-server.ts +++ b/packages/cli/src/utils/datatruck/repository-server.ts @@ -31,11 +31,6 @@ export type DatatruckRepositoryServerOptions = { }[]; }; -export type DatatruckServerOptions = { - log?: boolean; - repository?: DatatruckRepositoryServerOptions; -}; - export const headerKey = { user: "x-dtt-user", password: "x-dtt-password", @@ -107,7 +102,9 @@ const getRemoteAddress = ( export function createDatatruckRepositoryServer( options: Omit, - log?: boolean, + config: { + log?: boolean; + } = {}, ) { return createServer(async (req, res) => { try { @@ -125,7 +122,7 @@ export function createDatatruckRepositoryServer( return res.end(); } - if (log) console.info(`> ${req.url}`); + if (config.log) console.info(`> [repository] ${repository} - ${req.url}`); const fs = new LocalFs({ backend: backend.path, }); @@ -164,10 +161,10 @@ export function createDatatruckRepositoryServer( const json = await object(...params); if (json !== undefined) res.write(JSON.stringify(json)); } - if (log) console.info(`<${action}`); + if (config.log) console.info(`< [repository] ${repository} - ${action}`); res.end(); } catch (error) { - if (log) console.error(`<${req.url}`, error); + if (config.log) console.error(`< [repository] ${req.url}`, error); res.statusCode = 500; res.statusMessage = (error as Error).message; res.end(); diff --git a/packages/cli/src/utils/object.ts b/packages/cli/src/utils/object.ts index 19a6d20..e64e077 100644 --- a/packages/cli/src/utils/object.ts +++ b/packages/cli/src/utils/object.ts @@ -16,6 +16,15 @@ export function merge>( return target; } +export function omitProp, N extends keyof T>( + object: T, + name: N, +): Omit { + const result = { ...object }; + delete result[name]; + return result; +} + export function getErrorProperties(error: Error) { const alt: Record = {}; diff --git a/packages/cli/src/utils/schema.ts b/packages/cli/src/utils/schema.ts new file mode 100644 index 0000000..2dd9779 --- /dev/null +++ b/packages/cli/src/utils/schema.ts @@ -0,0 +1,80 @@ +import { omitProp } from "./object"; +import { JSONSchema7 } from "json-schema"; + +export function omitPropertySchema< + T extends { properties: Record }, + N extends keyof T["properties"], +>( + object: T, + name: N, +): Omit & { properties: Omit } { + return { + ...object, + properties: omitProp(object.properties, name as any), + }; +} + +type IfSchema< + KType extends string, + KValue extends string, + T extends string, + V extends JSONSchema7, +> = { + if: { + type: "object"; + properties: { + [k in KType]: { const: T }; + }; + }; + then: { + type: "object"; + properties: { + [k in KValue]: V; + }; + }; + else: false; +}; + +export function createCaseSchema< + KType extends string, + KValue extends string, + V extends { [K in KType]: JSONSchema7 }, +>( + keys: { type: KType; value: KValue }, + value: V, +): IfSchema[] { + return Object.entries(value).reduce( + (schemas, [type, value]) => { + schemas.push(createIfSchema(keys, type, value as JSONSchema7)); + return schemas; + }, + [] as IfSchema[], + ); +} + +export function createIfSchema< + KType extends string, + KValue extends string, + T extends string, + V extends JSONSchema7, +>( + keys: { type: KType; value: KValue }, + type: T, + value: V, +): IfSchema { + return { + if: { + type: "object", + properties: { + [keys.type]: { const: type }, + } as any, + }, + then: { + type: "object", + properties: { + [keys.value]: value, + } as any, + }, + else: false, + }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 96bf508..577606c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,9 @@ importers: commander: specifier: ^11.0.0 version: 11.0.0 + croner: + specifier: ^7.0.4 + version: 7.0.4 dayjs: specifier: ^1.11.10 version: 1.11.10 @@ -3171,6 +3174,11 @@ packages: dev: false optional: true + /croner@7.0.4: + resolution: {integrity: sha512-P8Zd88km8oQ0xH8Es0u75GtOnFyCNopuAhlFv5kAnbcTuXd0xNvRTgnxnJEs63FicCOsHTL7rpu4BHzY3cMq4w==} + engines: {node: '>=6.0'} + dev: false + /cross-spawn@5.1.0: resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} dependencies: