refactor progress

This commit is contained in:
Hampus Kraft
2026-02-17 12:22:36 +00:00
parent cb31608523
commit d5abd1a7e4
8257 changed files with 1190207 additions and 761040 deletions

View File

@@ -0,0 +1,74 @@
{
"name": "@fluxer/worker",
"version": "0.0.0",
"private": true,
"type": "module",
"exports": {
"./WorkerFactory": {
"import": "./src/runtime/WorkerFactory.tsx",
"types": "./src/runtime/WorkerFactory.tsx"
},
"./WorkerContext": {
"import": "./src/context/WorkerContext.tsx",
"types": "./src/context/WorkerContext.tsx"
},
"./WorkerTask": {
"import": "./src/contracts/WorkerTask.tsx",
"types": "./src/contracts/WorkerTask.tsx"
},
"./IWorkerService": {
"import": "./src/contracts/IWorkerService.tsx",
"types": "./src/contracts/IWorkerService.tsx"
},
"./WorkerTypes": {
"import": "./src/contracts/WorkerTypes.tsx",
"types": "./src/contracts/WorkerTypes.tsx"
},
"./IQueueProvider": {
"import": "./src/providers/IQueueProvider.tsx",
"types": "./src/providers/IQueueProvider.tsx"
},
"./HttpWorkerQueue": {
"import": "./src/providers/HttpWorkerQueue.tsx",
"types": "./src/providers/HttpWorkerQueue.tsx"
},
"./QueueProviderFactory": {
"import": "./src/providers/QueueProviderFactory.tsx",
"types": "./src/providers/QueueProviderFactory.tsx"
},
"./WorkerRunner": {
"import": "./src/runtime/WorkerRunner.tsx",
"types": "./src/runtime/WorkerRunner.tsx"
},
"./WorkerService": {
"import": "./src/services/WorkerService.tsx",
"types": "./src/services/WorkerService.tsx"
},
"./DirectQueueProvider": {
"import": "./src/providers/DirectQueueProvider.tsx",
"types": "./src/providers/DirectQueueProvider.tsx"
},
"./WorkerMetricsCollector": {
"import": "./src/services/WorkerMetricsCollector.tsx",
"types": "./src/services/WorkerMetricsCollector.tsx"
},
"./WorkerTaskRegistry": {
"import": "./src/runtime/WorkerTaskRegistry.tsx",
"types": "./src/runtime/WorkerTaskRegistry.tsx"
},
"./*": "./*"
},
"scripts": {
"typecheck": "tsgo --noEmit"
},
"dependencies": {
"@fluxer/constants": "workspace:*",
"@fluxer/logger": "workspace:*",
"@fluxer/queue": "workspace:*",
"itty-time": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
"@typescript/native-preview": "catalog:"
}
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
let workerDependencies: unknown | null = null;
export function setWorkerDependencies<T>(dependencies: T): void {
workerDependencies = dependencies;
}
export function getWorkerDependencies<T>(): T {
if (!workerDependencies) {
throw new Error('Worker dependencies have not been initialized. Call setWorkerDependencies() first.');
}
return workerDependencies as T;
}
export function hasWorkerDependencies(): boolean {
return workerDependencies !== null;
}
export function clearWorkerDependencies(): void {
workerDependencies = null;
}

View File

@@ -0,0 +1,32 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {WorkerJobOptions, WorkerJobPayload} from '@fluxer/worker/src/contracts/WorkerTypes';
export interface IWorkerService {
addJob<TPayload extends WorkerJobPayload = WorkerJobPayload>(
taskType: string,
payload: TPayload,
options?: WorkerJobOptions,
): Promise<void>;
cancelJob(jobId: string): Promise<boolean>;
retryDeadLetterJob(jobId: string): Promise<boolean>;
}

View File

@@ -0,0 +1,35 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {WorkerJobOptions, WorkerJobPayload} from '@fluxer/worker/src/contracts/WorkerTypes';
export interface WorkerTaskHelpers {
logger: LoggerInterface;
addJob: <TPayload extends WorkerJobPayload = WorkerJobPayload>(
taskType: string,
payload: TPayload,
options?: WorkerJobOptions,
) => Promise<void>;
}
export type WorkerTaskHandler<Payload = Record<string, unknown>> = (
payload: Payload,
helpers: WorkerTaskHelpers,
) => Promise<void>;

View File

@@ -0,0 +1,77 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
export type WorkerJobPayload = Record<string, unknown>;
export interface WorkerRuntimeConfig {
workerId?: string | undefined;
concurrency?: number | undefined;
taskTypes?: Array<string> | undefined;
}
export interface WorkerQueueConfig {
queueBaseUrl: string;
requestTimeoutMs?: number | undefined;
}
export interface WorkerConfig extends WorkerRuntimeConfig, WorkerQueueConfig {}
export interface TracingInterface {
withSpan<T>(options: {name: string; attributes?: Record<string, unknown>}, fn: () => Promise<T>): Promise<T>;
addSpanEvent(name: string, attributes?: Record<string, unknown>): void;
setSpanAttributes(attributes: Record<string, unknown>): void;
}
export interface MetricsServiceInterface {
gauge(options: {name: string; value: number; tags?: Record<string, string>}): void;
counter(options: {name: string; value: number; tags?: Record<string, string>}): void;
}
export interface QueueJob {
id: string;
task_type: string;
payload: WorkerJobPayload;
priority: number;
run_at: string;
created_at: string;
attempts: number;
max_attempts: number;
error?: string | null;
deduplication_id?: string | null;
}
export interface LeasedQueueJob {
receipt: string;
visibility_deadline: string;
job: QueueJob;
}
export interface EnqueueOptions {
runAt?: Date | undefined;
maxAttempts?: number | undefined;
priority?: number | undefined;
}
export interface WorkerJobOptions {
queueName?: string | undefined;
runAt?: Date | undefined;
maxAttempts?: number | undefined;
jobKey?: string | undefined;
priority?: number | undefined;
flags?: Array<string> | undefined;
}

View File

@@ -0,0 +1,162 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {CronScheduler} from '@fluxer/queue/src/cron/CronScheduler';
import type {QueueEngine} from '@fluxer/queue/src/engine/QueueEngine';
import type {JsonValue} from '@fluxer/queue/src/types/JsonTypes';
import type {
EnqueueOptions,
LeasedQueueJob,
TracingInterface,
WorkerJobPayload,
} from '@fluxer/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
export interface DirectQueueProviderOptions {
engine: QueueEngine;
cronScheduler: CronScheduler;
tracing?: TracingInterface;
}
export class DirectQueueProvider implements IQueueProvider {
private readonly engine: QueueEngine;
private readonly cronScheduler: CronScheduler;
private readonly tracing: TracingInterface | undefined;
constructor(options: DirectQueueProviderOptions) {
this.engine = options.engine;
this.cronScheduler = options.cronScheduler;
this.tracing = options.tracing;
}
private async withOptionalSpan<T>(
options: {name: string; attributes?: Record<string, unknown>},
fn: () => Promise<T>,
): Promise<T> {
if (this.tracing) {
return this.tracing.withSpan(options, fn);
}
return fn();
}
async enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise<string> {
return this.withOptionalSpan(
{
name: 'queue.enqueue',
attributes: {
'queue.task_type': taskType,
'queue.priority': options?.priority ?? 0,
'queue.max_attempts': options?.maxAttempts ?? 5,
'queue.scheduled': options?.runAt !== undefined,
'queue.provider': 'direct',
},
},
async () => {
const runAtMs = options?.runAt ? options.runAt.getTime() : null;
const result = await this.engine.enqueue(
taskType,
payload as JsonValue,
options?.priority ?? 0,
runAtMs,
options?.maxAttempts ?? 5,
null,
);
return result.job.id;
},
);
}
async dequeue(taskTypes: Array<string>, limit = 1): Promise<Array<LeasedQueueJob>> {
return this.withOptionalSpan(
{
name: 'queue.dequeue',
attributes: {
'queue.task_types': taskTypes.join(','),
'queue.limit': limit,
'queue.provider': 'direct',
},
},
async () => {
const leasedJobs = await this.engine.dequeue(taskTypes, limit, 0, null);
return leasedJobs.map(
(leasedJob): LeasedQueueJob => ({
receipt: leasedJob.receipt,
visibility_deadline: new Date(leasedJob.visibilityDeadlineMs).toISOString(),
job: {
id: leasedJob.job.id,
task_type: leasedJob.job.taskType,
payload: JSON.parse(Buffer.from(leasedJob.job.payload).toString('utf-8')) as WorkerJobPayload,
priority: leasedJob.job.priority,
run_at: new Date(leasedJob.job.runAtMs).toISOString(),
created_at: new Date(leasedJob.job.createdAtMs).toISOString(),
attempts: leasedJob.job.attempts,
max_attempts: leasedJob.job.maxAttempts,
error: leasedJob.job.error,
deduplication_id: leasedJob.job.deduplicationId,
},
}),
);
},
);
}
async upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise<void> {
await this.cronScheduler.upsert(id, taskType, payload as JsonValue, cronExpression);
}
async complete(receipt: string): Promise<void> {
return this.withOptionalSpan(
{
name: 'queue.complete',
attributes: {
'queue.receipt': receipt,
'queue.provider': 'direct',
},
},
async () => {
await this.engine.ack(receipt);
},
);
}
async fail(receipt: string, error: string): Promise<void> {
return this.withOptionalSpan(
{
name: 'queue.fail',
attributes: {
'queue.receipt': receipt,
'queue.error_message': error,
'queue.provider': 'direct',
},
},
async () => {
await this.engine.nack(receipt, error);
},
);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.engine.deleteJob(jobId);
}
async retryDeadLetterJob(jobId: string): Promise<boolean> {
const result = await this.engine.retryJob(jobId);
return result !== null;
}
}

View File

@@ -0,0 +1,261 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import {DEFAULT_HTTP_WORKER_TIMEOUT_MS} from '@fluxer/constants/src/Timeouts';
import type {
EnqueueOptions,
LeasedQueueJob,
TracingInterface,
WorkerJobPayload,
} from '@fluxer/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
export class HttpWorkerQueue implements IQueueProvider {
private readonly baseUrl: string;
private readonly timeoutMs: number;
private readonly tracing: TracingInterface | undefined;
constructor(options: {baseUrl: string; timeoutMs?: number | undefined; tracing?: TracingInterface | undefined}) {
this.baseUrl = options.baseUrl;
this.timeoutMs = options.timeoutMs ?? DEFAULT_HTTP_WORKER_TIMEOUT_MS;
this.tracing = options.tracing;
}
private createTimeoutController(): AbortController {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
(controller as {timeoutId?: NodeJS.Timeout}).timeoutId = timeoutId;
return controller;
}
private async fetchWithTimeout(input: string | URL, init?: RequestInit): Promise<Response> {
const controller = this.createTimeoutController();
try {
const response = await fetch(input, {
...init,
signal: controller.signal,
});
return response;
} finally {
const timeoutId = (controller as {timeoutId?: NodeJS.Timeout}).timeoutId;
if (timeoutId !== undefined) {
clearTimeout(timeoutId);
}
}
}
private async withOptionalSpan<T>(
options: {name: string; attributes?: Record<string, unknown>},
fn: () => Promise<T>,
): Promise<T> {
if (this.tracing) {
return this.tracing.withSpan(options, fn);
}
return fn();
}
private addSpanEvent(name: string, attributes?: Record<string, unknown>): void {
if (this.tracing) {
this.tracing.addSpanEvent(name, attributes);
}
}
private setSpanAttributes(attributes: Record<string, unknown>): void {
if (this.tracing) {
this.tracing.setSpanAttributes(attributes);
}
}
async enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise<string> {
return await this.withOptionalSpan(
{
name: 'queue.enqueue',
attributes: {
'queue.task_type': taskType,
'queue.priority': options?.priority ?? 0,
'queue.max_attempts': options?.maxAttempts ?? 5,
'queue.scheduled': options?.runAt !== undefined,
'net.peer.name': new URL(this.baseUrl).hostname,
},
},
async () => {
const body = {
task_type: taskType,
payload,
priority: options?.priority ?? 0,
run_at: options?.runAt?.toISOString(),
max_attempts: options?.maxAttempts ?? 5,
};
const response = await this.fetchWithTimeout(`${this.baseUrl}/enqueue`, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Failed to enqueue job: ${response.status} ${text}`);
}
const jobIdResult = (await response.json()) as {job_id: string};
this.setSpanAttributes({'queue.job_id': jobIdResult.job_id});
return jobIdResult.job_id;
},
);
}
async dequeue(taskTypes: Array<string>, limit = 1): Promise<Array<LeasedQueueJob>> {
return await this.withOptionalSpan(
{
name: 'queue.dequeue',
attributes: {
'queue.task_types': taskTypes.join(','),
'queue.limit': limit,
'queue.service': 'fluxer-queue',
},
},
async () => {
this.addSpanEvent('dequeue.start');
const url = new URL(`${this.baseUrl}/dequeue`);
url.searchParams.set('task_types', taskTypes.join(','));
url.searchParams.set('limit', limit.toString());
url.searchParams.set('wait_time_ms', '0');
const response = await this.fetchWithTimeout(url.toString(), {method: 'GET'});
if (!response.ok) {
const text = await response.text();
throw new Error(`Failed to dequeue job: ${response.status} ${text}`);
}
this.addSpanEvent('dequeue.parse_response');
const jobs = (await response.json()) as Array<LeasedQueueJob>;
const jobCount = jobs?.length ?? 0;
this.setSpanAttributes({
'queue.jobs_returned': jobCount,
'queue.empty': jobCount === 0,
});
this.addSpanEvent('dequeue.complete');
return jobs ?? [];
},
);
}
async upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise<void> {
const response = await this.fetchWithTimeout(`${this.baseUrl}/cron`, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({id, task_type: taskType, payload, cron_expression: cronExpression}),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Failed to upsert cron job: ${response.status} ${text}`);
}
}
async complete(receipt: string): Promise<void> {
return await this.withOptionalSpan(
{
name: 'queue.complete',
attributes: {
'queue.receipt': receipt,
},
},
async () => {
const response = await this.fetchWithTimeout(`${this.baseUrl}/ack`, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({receipt}),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Failed to complete job: ${response.status} ${text}`);
}
},
);
}
async fail(receipt: string, error: string): Promise<void> {
return await this.withOptionalSpan(
{
name: 'queue.fail',
attributes: {
'queue.receipt': receipt,
'queue.error_message': error,
},
},
async () => {
const response = await this.fetchWithTimeout(`${this.baseUrl}/nack`, {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({receipt, error}),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Failed to fail job: ${response.status} ${text}`);
}
},
);
}
async cancelJob(jobId: string): Promise<boolean> {
const response = await this.fetchWithTimeout(`${this.baseUrl}/job/${jobId}`, {
method: 'DELETE',
});
if (!response.ok) {
const text = await response.text();
if (response.status === 404) {
return false;
}
throw new Error(`Failed to cancel job: ${response.status} ${text}`);
}
const result = (await response.json()) as {success: boolean};
return result.success ?? true;
}
async retryDeadLetterJob(jobId: string): Promise<boolean> {
const response = await this.fetchWithTimeout(`${this.baseUrl}/retry/${jobId}`, {
method: 'POST',
});
if (!response.ok) {
const text = await response.text();
if (response.status === 404) {
return false;
}
throw new Error(`Failed to retry job: ${response.status} ${text}`);
}
return true;
}
}

View File

@@ -0,0 +1,30 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {EnqueueOptions, LeasedQueueJob, WorkerJobPayload} from '@fluxer/worker/src/contracts/WorkerTypes';
export interface IQueueProvider {
enqueue(taskType: string, payload: WorkerJobPayload, options?: EnqueueOptions): Promise<string>;
dequeue(taskTypes: Array<string>, limit?: number): Promise<Array<LeasedQueueJob>>;
upsertCron(id: string, taskType: string, payload: WorkerJobPayload, cronExpression: string): Promise<void>;
complete(receipt: string): Promise<void>;
fail(receipt: string, error: string): Promise<void>;
cancelJob(jobId: string): Promise<boolean>;
retryDeadLetterJob(jobId: string): Promise<boolean>;
}

View File

@@ -0,0 +1,45 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {TracingInterface} from '@fluxer/worker/src/contracts/WorkerTypes';
import {HttpWorkerQueue} from '@fluxer/worker/src/providers/HttpWorkerQueue';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
export interface QueueProviderFactoryOptions {
queueProvider?: IQueueProvider | undefined;
queueBaseUrl?: string | undefined;
timeoutMs?: number | undefined;
tracing?: TracingInterface | undefined;
}
export function createQueueProvider(options: QueueProviderFactoryOptions): IQueueProvider {
if (options.queueProvider) {
return options.queueProvider;
}
if (!options.queueBaseUrl) {
throw new Error('Queue provider requires either queueProvider or queueBaseUrl');
}
return new HttpWorkerQueue({
baseUrl: options.queueBaseUrl,
timeoutMs: options.timeoutMs,
tracing: options.tracing,
});
}

View File

@@ -0,0 +1,205 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import {setWorkerDependencies} from '@fluxer/worker/src/context/WorkerContext';
import type {IWorkerService} from '@fluxer/worker/src/contracts/IWorkerService';
import type {WorkerTaskHandler} from '@fluxer/worker/src/contracts/WorkerTask';
import type {
LeasedQueueJob,
TracingInterface,
WorkerConfig,
WorkerQueueConfig,
WorkerRuntimeConfig,
} from '@fluxer/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
import {WorkerRunner} from '@fluxer/worker/src/runtime/WorkerRunner';
import {WorkerTaskRegistry} from '@fluxer/worker/src/runtime/WorkerTaskRegistry';
import {WorkerService} from '@fluxer/worker/src/services/WorkerService';
export interface CreateWorkerOptions {
queue: WorkerQueueOptions;
runtime?: WorkerRuntimeConfig | undefined;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface CreateWorkerLegacyOptions {
config: WorkerConfig;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface WorkerQueueOptions {
queueProvider?: IQueueProvider | undefined;
queueBaseUrl?: string | undefined;
requestTimeoutMs?: number | undefined;
}
interface ResolvedWorkerFactoryOptions {
queue: WorkerQueueOptions;
runtime: WorkerRuntimeConfig;
logger: LoggerInterface;
dependencies?: unknown;
taskRegistry?: WorkerTaskRegistry | undefined;
tracing?: TracingInterface | undefined;
}
export interface WorkerResult {
start: () => Promise<void>;
shutdown: () => Promise<void>;
processTask: (job: LeasedQueueJob) => Promise<void>;
getRunner: () => WorkerRunner;
getWorkerService: () => IWorkerService;
registerTask: <TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>) => void;
registerTasks: (tasks: Record<string, WorkerTaskHandler>) => void;
}
type WorkerFactoryOptions = CreateWorkerOptions | CreateWorkerLegacyOptions;
function isLegacyCreateWorkerOptions(options: WorkerFactoryOptions): options is CreateWorkerLegacyOptions {
return 'config' in options;
}
function resolveLegacyQueueOptions(config: WorkerQueueConfig, queueProvider?: IQueueProvider): WorkerQueueOptions {
return {
queueProvider,
queueBaseUrl: config.queueBaseUrl,
requestTimeoutMs: config.requestTimeoutMs,
};
}
function resolveWorkerFactoryOptions(options: WorkerFactoryOptions): ResolvedWorkerFactoryOptions {
if (isLegacyCreateWorkerOptions(options)) {
return {
queue: resolveLegacyQueueOptions(options.config, options.queueProvider),
runtime: {
workerId: options.config.workerId,
taskTypes: options.config.taskTypes,
concurrency: options.config.concurrency,
},
logger: options.logger,
dependencies: options.dependencies,
taskRegistry: options.taskRegistry,
tracing: options.tracing,
};
}
return {
queue: options.queue,
runtime: options.runtime ?? {},
logger: options.logger,
dependencies: options.dependencies,
taskRegistry: options.taskRegistry,
tracing: options.tracing,
};
}
function assertTaskRegistryMutable(runner: WorkerRunner | null): void {
if (runner?.isRunning()) {
throw new Error('Cannot register tasks after worker start. Register tasks before starting the worker.');
}
}
export function createWorker(options: WorkerFactoryOptions): WorkerResult {
const resolvedOptions = resolveWorkerFactoryOptions(options);
const {queue, runtime, logger, dependencies, taskRegistry: providedRegistry, tracing} = resolvedOptions;
if (dependencies !== undefined) {
setWorkerDependencies(dependencies);
}
const taskRegistry = providedRegistry ?? new WorkerTaskRegistry();
let runner: WorkerRunner | null = null;
let workerService: WorkerService | null = null;
function ensureRunner(): WorkerRunner {
if (!runner) {
runner = new WorkerRunner({
tasks: taskRegistry.getTasks(),
queueBaseUrl: queue.queueBaseUrl,
queueProvider: queue.queueProvider,
logger,
workerId: runtime.workerId,
taskTypes: runtime.taskTypes,
concurrency: runtime.concurrency,
tracing,
requestTimeoutMs: queue.requestTimeoutMs,
});
}
return runner;
}
function ensureWorkerService(): WorkerService {
if (!workerService) {
workerService = new WorkerService({
queueBaseUrl: queue.queueBaseUrl,
queueProvider: queue.queueProvider,
logger,
tracing,
timeoutMs: queue.requestTimeoutMs,
});
}
return workerService;
}
return {
async start() {
const r = ensureRunner();
await r.start();
},
async shutdown() {
if (runner) {
await runner.stop();
}
},
async processTask(job: LeasedQueueJob) {
const r = ensureRunner();
await r.processJob(job);
},
getRunner() {
return ensureRunner();
},
getWorkerService() {
return ensureWorkerService();
},
registerTask<TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>) {
assertTaskRegistryMutable(runner);
taskRegistry.register(name, handler);
runner = null;
},
registerTasks(tasks: Record<string, WorkerTaskHandler>) {
assertTaskRegistryMutable(runner);
taskRegistry.registerAll(tasks);
runner = null;
},
};
}

View File

@@ -0,0 +1,224 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import {randomUUID} from 'node:crypto';
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {WorkerTaskHandler} from '@fluxer/worker/src/contracts/WorkerTask';
import type {LeasedQueueJob, TracingInterface} from '@fluxer/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
import {createQueueProvider} from '@fluxer/worker/src/providers/QueueProviderFactory';
import {WorkerService} from '@fluxer/worker/src/services/WorkerService';
import {ms} from 'itty-time';
export interface WorkerRunnerOptions {
tasks: Record<string, WorkerTaskHandler>;
queueBaseUrl?: string | undefined;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
workerId?: string | undefined;
taskTypes?: Array<string> | undefined;
concurrency?: number | undefined;
tracing?: TracingInterface | undefined;
requestTimeoutMs?: number | undefined;
}
export class WorkerRunner {
private readonly tasks: Record<string, WorkerTaskHandler>;
private readonly workerId: string;
private readonly taskTypes: Array<string>;
private readonly concurrency: number;
private readonly queue: IQueueProvider;
private readonly workerService: WorkerService;
private readonly logger: LoggerInterface;
private readonly tracing: TracingInterface | undefined;
private running = false;
private abortController: AbortController | null = null;
private workerLoopPromises: Array<Promise<void>> = [];
constructor(options: WorkerRunnerOptions) {
this.tasks = options.tasks;
this.workerId = options.workerId ?? `worker-${randomUUID()}`;
this.taskTypes = options.taskTypes ?? Object.keys(options.tasks);
this.concurrency = options.concurrency ?? 1;
this.logger = options.logger;
this.tracing = options.tracing;
this.queue = createQueueProvider({
queueProvider: options.queueProvider,
queueBaseUrl: options.queueBaseUrl,
timeoutMs: options.requestTimeoutMs,
tracing: options.tracing,
});
this.workerService = new WorkerService({
queueProvider: this.queue,
logger: options.logger,
});
}
async start(): Promise<void> {
if (this.running) {
this.logger.warn({workerId: this.workerId}, 'Worker already running');
return;
}
this.running = true;
this.abortController = new AbortController();
this.logger.info(
{workerId: this.workerId, taskTypes: this.taskTypes, concurrency: this.concurrency},
'Worker starting',
);
this.workerLoopPromises = Array.from({length: this.concurrency}, (_, i) =>
this.workerLoop(i, this.abortController!.signal),
);
Promise.all(this.workerLoopPromises).catch((error) => {
this.logger.error({workerId: this.workerId, error}, 'Worker loop failed unexpectedly');
});
}
async stop(): Promise<void> {
if (!this.running) {
return;
}
this.running = false;
this.abortController?.abort();
const stopTimeout = new Promise<void>((resolve) => setTimeout(resolve, ms('2 seconds')));
await Promise.race([Promise.all(this.workerLoopPromises), stopTimeout]);
this.workerLoopPromises = [];
this.logger.info({workerId: this.workerId}, 'Worker stopped');
}
async processJob(leasedJob: LeasedQueueJob): Promise<void> {
await this.executeJob(leasedJob);
}
getWorkerService(): WorkerService {
return this.workerService;
}
getQueue(): IQueueProvider {
return this.queue;
}
isRunning(): boolean {
return this.running;
}
private async workerLoop(workerIndex: number, signal: AbortSignal): Promise<void> {
this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop started');
while (!signal.aborted) {
try {
const leasedJobs = await this.queue.dequeue(this.taskTypes, 1);
if (!leasedJobs || leasedJobs.length === 0) {
await this.sleep(100);
continue;
}
const leasedJob = leasedJobs[0]!;
const job = leasedJob.job;
this.logger.info(
{
workerId: this.workerId,
workerIndex,
jobId: job.id,
taskType: job.task_type,
attempts: job.attempts,
receipt: leasedJob.receipt,
},
'Processing job',
);
await this.executeJob(leasedJob);
this.logger.info({workerId: this.workerId, workerIndex, jobId: job.id}, 'Job completed successfully');
} catch (error) {
this.logger.error({workerId: this.workerId, workerIndex, error}, 'Worker loop error');
await this.sleep(ms('1 second'));
}
}
this.logger.info({workerId: this.workerId, workerIndex}, 'Worker loop stopped');
}
private async executeJob(leasedJob: LeasedQueueJob): Promise<void> {
const execute = async () => {
const task = this.tasks[leasedJob.job.task_type];
if (!task) {
throw new Error(`Unknown task: ${leasedJob.job.task_type}`);
}
this.tracing?.addSpanEvent('job.execution.start');
try {
await task(leasedJob.job.payload as never, {
logger: this.logger.child({jobId: leasedJob.job.id}),
addJob: this.workerService.addJob.bind(this.workerService),
});
this.tracing?.addSpanEvent('job.execution.success');
this.tracing?.setSpanAttributes({'job.status': 'success'});
await this.queue.complete(leasedJob.receipt);
} catch (error) {
this.logger.error({jobId: leasedJob.job.id, error}, 'Job failed');
this.tracing?.setSpanAttributes({
'job.status': 'failed',
'job.error': error instanceof Error ? error.message : String(error),
});
this.tracing?.addSpanEvent('job.execution.failed', {
error: error instanceof Error ? error.message : String(error),
});
await this.queue.fail(leasedJob.receipt, String(error));
}
};
if (this.tracing) {
await this.tracing.withSpan(
{
name: 'worker.process_job',
attributes: {
'worker.id': this.workerId,
'job.id': leasedJob.job.id,
'job.task_type': leasedJob.job.task_type,
'job.attempts': leasedJob.job.attempts,
},
},
execute,
);
} else {
await execute();
}
}
private async sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
}

View File

@@ -0,0 +1,60 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {WorkerTaskHandler} from '@fluxer/worker/src/contracts/WorkerTask';
export class WorkerTaskRegistry {
private readonly tasks: Map<string, WorkerTaskHandler> = new Map();
register<TPayload = Record<string, unknown>>(name: string, handler: WorkerTaskHandler<TPayload>): this {
this.tasks.set(name, handler as WorkerTaskHandler);
return this;
}
registerAll(tasks: Record<string, WorkerTaskHandler>): this {
for (const [name, handler] of Object.entries(tasks)) {
this.tasks.set(name, handler);
}
return this;
}
get(name: string): WorkerTaskHandler | undefined {
return this.tasks.get(name);
}
has(name: string): boolean {
return this.tasks.has(name);
}
getTaskNames(): Array<string> {
return Array.from(this.tasks.keys());
}
getTasks(): Record<string, WorkerTaskHandler> {
return Object.fromEntries(this.tasks);
}
get size(): number {
return this.tasks.size;
}
}
export function createTaskRegistry(): WorkerTaskRegistry {
return new WorkerTaskRegistry();
}

View File

@@ -0,0 +1,170 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {MetricsServiceInterface} from '@fluxer/worker/src/contracts/WorkerTypes';
import {ms} from 'itty-time';
export interface QueueWithSize {
getQueueSize(): Promise<number>;
}
export interface KVClientInterface {
health(): Promise<boolean>;
}
export interface WorkerMetricsCollectorOptions {
kvClient: KVClientInterface;
metricsService: MetricsServiceInterface;
logger: LoggerInterface;
queues?: {
assetDeletion?: QueueWithSize;
cloudflarePurge?: QueueWithSize;
bulkMessageDeletion?: QueueWithSize;
accountDeletion?: QueueWithSize;
};
reportIntervalMs?: number;
}
export class WorkerMetricsCollector {
private readonly kvClient: KVClientInterface;
private readonly metricsService: MetricsServiceInterface;
private readonly logger: LoggerInterface;
private readonly queues: {
assetDeletion?: QueueWithSize;
cloudflarePurge?: QueueWithSize;
bulkMessageDeletion?: QueueWithSize;
accountDeletion?: QueueWithSize;
};
private readonly reportIntervalMs: number;
private intervalHandle: ReturnType<typeof setInterval> | null = null;
private kvErrorCount = 0;
constructor(options: WorkerMetricsCollectorOptions) {
this.kvClient = options.kvClient;
this.metricsService = options.metricsService;
this.logger = options.logger;
this.queues = options.queues ?? {};
this.reportIntervalMs = options.reportIntervalMs ?? ms('30 seconds');
}
start(): void {
if (this.intervalHandle) return;
this.logger.info({intervalMs: this.reportIntervalMs}, 'Starting WorkerMetricsCollector');
this.collectAndReport().catch((err) => {
this.logger.error({err}, 'Initial metrics collection failed');
});
this.intervalHandle = setInterval(() => {
this.collectAndReport().catch((err) => {
this.logger.error({err}, 'Metrics collection failed');
});
}, this.reportIntervalMs);
}
stop(): void {
if (this.intervalHandle) {
clearInterval(this.intervalHandle);
this.intervalHandle = null;
this.logger.info({}, 'Stopped WorkerMetricsCollector');
}
}
private async collectAndReport(): Promise<void> {
const [kvQueueSizes, kvConnectionStatus] = await Promise.all([
this.collectKVQueueSizes(),
this.collectKVConnectionStatus(),
]);
this.reportKVQueueSizes(kvQueueSizes);
this.reportKVHealthMetrics(kvConnectionStatus);
}
private async collectKVQueueSizes(): Promise<{
assetDeletion: number;
cloudflarePurge: number;
bulkMessageDeletion: number;
accountDeletion: number;
}> {
try {
const [assetDeletion, cloudflarePurge, bulkMessageDeletion, accountDeletion] = await Promise.all([
this.queues.assetDeletion?.getQueueSize() ?? Promise.resolve(0),
this.queues.cloudflarePurge?.getQueueSize() ?? Promise.resolve(0),
this.queues.bulkMessageDeletion?.getQueueSize() ?? Promise.resolve(0),
this.queues.accountDeletion?.getQueueSize() ?? Promise.resolve(0),
]);
return {assetDeletion, cloudflarePurge, bulkMessageDeletion, accountDeletion};
} catch (err) {
this.kvErrorCount++;
this.logger.error({err}, 'Failed to collect KV queue sizes');
return {assetDeletion: 0, cloudflarePurge: 0, bulkMessageDeletion: 0, accountDeletion: 0};
}
}
private reportKVQueueSizes(sizes: {
assetDeletion: number;
cloudflarePurge: number;
bulkMessageDeletion: number;
accountDeletion: number;
}): void {
this.metricsService.gauge({
name: 'worker.kv_queue.asset_deletion',
value: sizes.assetDeletion,
});
this.metricsService.gauge({
name: 'worker.kv_queue.cloudflare_purge',
value: sizes.cloudflarePurge,
});
this.metricsService.gauge({
name: 'worker.kv_queue.bulk_message_deletion',
value: sizes.bulkMessageDeletion,
});
this.metricsService.gauge({
name: 'worker.kv_queue.account_deletion',
value: sizes.accountDeletion,
});
}
private async collectKVConnectionStatus(): Promise<boolean> {
try {
return await this.kvClient.health();
} catch (err) {
this.kvErrorCount++;
this.logger.error({err}, 'KV health check failed');
return false;
}
}
private reportKVHealthMetrics(isConnected: boolean): void {
this.metricsService.gauge({
name: 'kv.connection.status',
value: isConnected ? 1 : 0,
});
if (this.kvErrorCount > 0) {
this.metricsService.counter({
name: 'kv.command.error',
value: this.kvErrorCount,
});
this.kvErrorCount = 0;
}
}
}

View File

@@ -0,0 +1,99 @@
/*
* Copyright (C) 2026 Fluxer Contributors
*
* This file is part of Fluxer.
*
* Fluxer is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Fluxer is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
*/
import type {LoggerInterface} from '@fluxer/logger/src/LoggerInterface';
import type {IWorkerService} from '@fluxer/worker/src/contracts/IWorkerService';
import type {TracingInterface, WorkerJobOptions, WorkerJobPayload} from '@fluxer/worker/src/contracts/WorkerTypes';
import type {IQueueProvider} from '@fluxer/worker/src/providers/IQueueProvider';
import {createQueueProvider} from '@fluxer/worker/src/providers/QueueProviderFactory';
export interface WorkerServiceOptions {
queueBaseUrl?: string | undefined;
queueProvider?: IQueueProvider | undefined;
logger: LoggerInterface;
tracing?: TracingInterface | undefined;
timeoutMs?: number | undefined;
}
export class WorkerService implements IWorkerService {
private readonly queue: IQueueProvider;
private readonly logger: LoggerInterface;
constructor(options: WorkerServiceOptions) {
this.queue = createQueueProvider({
queueProvider: options.queueProvider,
queueBaseUrl: options.queueBaseUrl,
timeoutMs: options.timeoutMs,
tracing: options.tracing,
});
this.logger = options.logger;
}
async addJob<TPayload extends WorkerJobPayload = WorkerJobPayload>(
taskType: string,
payload: TPayload,
options?: WorkerJobOptions,
): Promise<void> {
try {
await this.queue.enqueue(taskType, payload, {
runAt: options?.runAt,
maxAttempts: options?.maxAttempts,
priority: options?.priority,
});
this.logger.debug({taskType, payload}, 'Job queued successfully');
} catch (error) {
this.logger.error({error, taskType, payload}, 'Failed to queue job');
throw error;
}
}
async cancelJob(jobId: string): Promise<boolean> {
try {
const cancelled = await this.queue.cancelJob(jobId);
if (cancelled) {
this.logger.info({jobId}, 'Job cancelled successfully');
} else {
this.logger.debug({jobId}, 'Job not found (may have already been processed)');
}
return cancelled;
} catch (error) {
this.logger.error({error, jobId}, 'Failed to cancel job');
throw error;
}
}
async retryDeadLetterJob(jobId: string): Promise<boolean> {
try {
const retried = await this.queue.retryDeadLetterJob(jobId);
if (retried) {
this.logger.info({jobId}, 'Dead letter job retried successfully');
} else {
this.logger.debug({jobId}, 'Job not found in dead letter queue');
}
return retried;
} catch (error) {
this.logger.error({error, jobId}, 'Failed to retry dead letter job');
throw error;
}
}
getQueue(): IQueueProvider {
return this.queue;
}
}

View File

@@ -0,0 +1,5 @@
{
"extends": "../../tsconfigs/package.json",
"compilerOptions": {},
"include": ["src"]
}