refactor progress

fluxer_queue/Dockerfile

@@ -0,0 +1,72 @@
+ARG BUILD_SHA
+ARG BUILD_NUMBER
+ARG BUILD_TIMESTAMP
+ARG RELEASE_CHANNEL=nightly
+
+FROM node:24-bookworm-slim AS base
+
+WORKDIR /usr/src/app
+
+RUN corepack enable && corepack prepare pnpm@10.26.0 --activate
+
+FROM base AS generator
+
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY patches/ ./patches/
+COPY packages/config/ ./packages/config/
+COPY packages/constants/ ./packages/constants/
+
+RUN pnpm install --frozen-lockfile --filter @fluxer/config...
+RUN pnpm --filter @fluxer/config generate
+
+FROM base AS deps
+
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY patches/ ./patches/
+COPY packages/ ./packages/
+COPY fluxer_queue/package.json ./fluxer_queue/
+
+RUN pnpm install --frozen-lockfile --prod
+
+FROM node:24-bookworm-slim
+
+ARG BUILD_SHA
+ARG BUILD_NUMBER
+ARG BUILD_TIMESTAMP
+ARG RELEASE_CHANNEL
+
+WORKDIR /usr/src/app/fluxer_queue
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	curl && \
+	rm -rf /var/lib/apt/lists/*
+
+RUN corepack enable && corepack prepare pnpm@10.26.0 --activate
+
+COPY --from=deps /usr/src/app/node_modules /usr/src/app/node_modules
+COPY --from=deps /usr/src/app/fluxer_queue/node_modules ./node_modules
+COPY --from=deps /usr/src/app/packages /usr/src/app/packages
+COPY --from=generator /usr/src/app/packages/config/src/ConfigSchema.json /usr/src/app/packages/config/src/ConfigSchema.json
+COPY --from=generator /usr/src/app/packages/config/src/MasterZodSchema.generated.tsx /usr/src/app/packages/config/src/MasterZodSchema.generated.tsx
+COPY tsconfigs /usr/src/app/tsconfigs
+COPY fluxer_queue/package.json ./
+COPY fluxer_queue/tsconfig.json ./
+COPY fluxer_queue/src ./src
+
+RUN mkdir -p /usr/src/app/.cache/corepack && \
+	chown -R nobody:nogroup /usr/src/app
+RUN mkdir -p /data && chown -R nobody:nogroup /data
+
+ENV HOME=/usr/src/app
+ENV COREPACK_HOME=/usr/src/app/.cache/corepack
+ENV NODE_ENV=production
+ENV BUILD_SHA=${BUILD_SHA}
+ENV BUILD_NUMBER=${BUILD_NUMBER}
+ENV BUILD_TIMESTAMP=${BUILD_TIMESTAMP}
+ENV RELEASE_CHANNEL=${RELEASE_CHANNEL}
+
+USER nobody
+
+EXPOSE 8080
+
+CMD ["pnpm", "exec", "tsx", "src/App.tsx"]

fluxer_queue/package.json

@@ -0,0 +1,31 @@
+{
+	"name": "fluxer_queue",
+	"version": "0.0.0",
+	"description": "Distributed job queue service with persistence, cron scheduling, and observability",
+	"license": "AGPL-3.0-or-later",
+	"type": "module",
+	"main": "dist/App.js",
+	"scripts": {
+		"build": "tsgo",
+		"dev": "tsx watch --clear-screen=false src/App.tsx",
+		"start": "node dist/App.js",
+		"typecheck": "tsgo --noEmit"
+	},
+	"dependencies": {
+		"@fluxer/config": "workspace:*",
+		"@fluxer/hono": "workspace:*",
+		"@fluxer/initialization": "workspace:*",
+		"@fluxer/logger": "workspace:*",
+		"@fluxer/queue": "workspace:*",
+		"@fluxer/rate_limit": "workspace:*",
+		"@fluxer/sentry": "workspace:*",
+		"tsx": "catalog:"
+	},
+	"devDependencies": {
+		"@types/node": "catalog:",
+		"@typescript/native-preview": "catalog:"
+	},
+	"engines": {
+		"node": ">=24.0.0"
+	}
+}

fluxer_queue/src/App.tsx

@@ -0,0 +1,110 @@
+/*
+ * Copyright (C) 2026 Fluxer Contributors
+ *
+ * This file is part of Fluxer.
+ *
+ * Fluxer is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Fluxer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {Config} from '@app/Config';
+import {shutdownInstrumentation} from '@app/Instrument';
+import {Logger} from '@app/Logger';
+import {createServiceTelemetry} from '@fluxer/hono/src/middleware/TelemetryAdapters';
+import {createServer, setupGracefulShutdown} from '@fluxer/hono/src/Server';
+import {createQueueApp} from '@fluxer/queue/src/App';
+import {createRateLimitService, type RateLimitService} from '@fluxer/rate_limit/src/RateLimitService';
+import {captureException} from '@fluxer/sentry/src/Sentry';
+import '@app/Instrument';
+
+async function main() {
+	Logger.info('Starting fluxer_queue');
+
+	const telemetry = createServiceTelemetry({
+		serviceName: 'fluxer-queue',
+		skipPaths: ['/_health'],
+	});
+
+	function createLogger(name: string) {
+		return Logger.child({module: name});
+	}
+
+	let rateLimitService: RateLimitService | null = null;
+	rateLimitService = createRateLimitService(null);
+
+	const {app, engine, cronScheduler, start, shutdown} = createQueueApp({
+		config: {
+			dataDir: Config.dataDir,
+			snapshotEveryMs: Config.snapshotEveryMs,
+			snapshotAfterOps: Config.snapshotAfterOps,
+			snapshotZstdLevel: Config.snapshotZstdLevel,
+			defaultVisibilityTimeoutMs: Config.defaultVisibilityTimeoutMs,
+			visibilityTimeoutBackoffMs: Config.visibilityTimeoutBackoffMs,
+			maxReceiveBatch: Config.maxReceiveBatch,
+			commandBuffer: Config.commandBuffer,
+		},
+		loggerFactory: createLogger,
+		metricsCollector: telemetry.metricsCollector,
+		tracing: telemetry.tracing,
+		rateLimitService,
+		rateLimitConfig:
+			Config.rateLimit?.limit != null && Config.rateLimit.window_ms != null
+				? {
+						enabled: true,
+						maxAttempts: Config.rateLimit.limit,
+						windowMs: Config.rateLimit.window_ms,
+						skipPaths: ['/_health'],
+					}
+				: null,
+		internalSecret: Config.secret,
+	});
+
+	await start();
+
+	app.use('*', async (_ctx, next) => {
+		const stats = engine.getStats();
+		const cronStats = cronScheduler.getStats();
+
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_jobs_ready', value: stats.ready});
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_jobs_processing', value: stats.processing});
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_jobs_scheduled', value: stats.scheduled});
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_jobs_dead_letter', value: stats.deadLetter});
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_cron_schedules_total', value: cronStats.total});
+		telemetry.metricsCollector.recordGauge({name: 'fluxer_queue_cron_schedules_enabled', value: cronStats.enabled});
+
+		await next();
+	});
+
+	const server = createServer(app, {port: Config.port});
+
+	setupGracefulShutdown(
+		async () => {
+			await shutdown();
+			await shutdownInstrumentation();
+			await new Promise<void>((resolve) => {
+				server.close(() => resolve());
+			});
+		},
+		{logger: Logger},
+	);
+
+	Logger.info({port: Config.port}, 'Server started');
+
+	await new Promise(() => {});
+}
+
+main().catch((err) => {
+	Logger.error({err}, 'Fatal error');
+	captureException(err instanceof Error ? err : new Error(String(err)), {fatal: true});
+	process.exit(1);
+});

fluxer_queue/src/Config.tsx

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2026 Fluxer Contributors
+ *
+ * This file is part of Fluxer.
+ *
+ * Fluxer is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Fluxer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {loadConfig} from '@fluxer/config/src/ConfigLoader';
+import {extractBaseServiceConfig} from '@fluxer/config/src/ServiceConfigSlices';
+
+const master = await loadConfig();
+
+export const Config = {
+	...extractBaseServiceConfig(master),
+	port: master.services.queue.port,
+	dataDir: master.services.queue.data_dir,
+	snapshotEveryMs: master.services.queue.snapshot_every_ms,
+	snapshotAfterOps: master.services.queue.snapshot_after_ops,
+	snapshotZstdLevel: master.services.queue.snapshot_zstd_level,
+	defaultVisibilityTimeoutMs: master.services.queue.default_visibility_timeout_ms,
+	visibilityTimeoutBackoffMs: master.services.queue.visibility_timeout_backoff_ms,
+	maxReceiveBatch: master.services.queue.max_receive_batch,
+	commandBuffer: master.services.queue.command_buffer,
+	rateLimit: master.services.queue.rate_limit ?? null,
+	secret: master.services.queue.secret,
+};
+
+export type Config = typeof Config;

fluxer_queue/src/Instrument.tsx

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2026 Fluxer Contributors
+ *
+ * This file is part of Fluxer.
+ *
+ * Fluxer is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Fluxer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {Config} from '@app/Config';
+import {createServiceInstrumentation} from '@fluxer/initialization/src/CreateServiceInstrumentation';
+
+export const shutdownInstrumentation = createServiceInstrumentation({
+	serviceName: 'fluxer-queue',
+	config: Config,
+});

fluxer_queue/src/Logger.tsx

@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2026 Fluxer Contributors
+ *
+ * This file is part of Fluxer.
+ *
+ * Fluxer is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Fluxer is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Fluxer. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import {createLogger, type Logger as FluxerLogger} from '@fluxer/logger/src/Logger';
+
+export const Logger = createLogger('fluxer-queue');
+export type Logger = FluxerLogger;

fluxer_queue/tsconfig.json

@@ -0,0 +1,10 @@
+{
+	"extends": "../tsconfigs/hono-service.json",
+	"compilerOptions": {
+		"paths": {
+			"@app/*": ["./src/*"],
+			"@fluxer/*": ["./../packages/*", "./../packages/*/src/index.tsx"]
+		}
+	},
+	"include": ["src/**/*"]
+}