fix: various fixes to sentry-reported errors and more

2026-02-18 15:38:51 +00:00
parent 302c0d2a0c
commit 0517a966a3
357 changed files with 25420 additions and 16281 deletions
--- a/fluxer_server/src/HealthCheck.tsx
+++ b/fluxer_server/src/HealthCheck.tsx
@@ -36,7 +36,7 @@ export interface HealthCheckResponse {
 	services: {
 		kv: ServiceHealth;
 		s3: ServiceHealth;
-		queue: ServiceHealth;
+		jetstream: ServiceHealth;
 		mediaProxy: ServiceHealth;
 		admin: ServiceHealth;
 		api: ServiceHealth;
@@ -123,38 +123,19 @@ async function checkS3Health(services: InitializedServices, latencyThresholdMs:
 	}
 }

-async function checkQueueHealth(services: InitializedServices, latencyThresholdMs: number): Promise<ServiceHealth> {
-	if (services.queue === undefined) {
+function checkJetStreamHealth(services: InitializedServices): ServiceHealth {
+	if (services.jsConnectionManager === undefined) {
 		return {status: 'disabled'};
 	}

-	try {
-		const start = Date.now();
-		const engine = services.queue.engine;
-		const stats = engine.getStats();
-		const details: Record<string, unknown> = {...stats};
-		const latencyMs = Date.now() - start;
-
-		if (latencyMs > latencyThresholdMs) {
-			return {
-				status: 'degraded',
-				latencyMs,
-				message: 'High latency detected',
-				details,
-			};
-		}
-
-		return {
-			status: 'healthy',
-			latencyMs,
-			details,
-		};
-	} catch (error) {
+	if (services.jsConnectionManager.isClosed()) {
 		return {
 			status: 'unhealthy',
-			message: error instanceof Error ? error.message : 'Unknown error',
+			message: 'JetStream connection is closed',
 		};
 	}
+
+	return {status: 'healthy'};
 }

 async function checkMediaProxyHealth(services: InitializedServices): Promise<ServiceHealth> {
@@ -224,7 +205,7 @@ export function createHealthCheckHandler(config: HealthCheckConfig) {
 		const healthChecks: HealthCheckResponse['services'] = {
 			kv: await checkKVHealth(services, latencyThresholdMs),
 			s3: await checkS3Health(services, latencyThresholdMs),
-			queue: await checkQueueHealth(services, latencyThresholdMs),
+			jetstream: checkJetStreamHealth(services),
 			mediaProxy: await checkMediaProxyHealth(services),
 			admin: await checkAdminHealth(services),
 			api: await checkAPIHealth(services),
@@ -253,7 +234,7 @@ export interface ReadinessCheckResponse {
 		database?: {ready: boolean; message?: string};
 		kv?: {ready: boolean; message?: string};
 		s3?: {ready: boolean; message?: string};
-		queue?: {ready: boolean; message?: string};
+		jetstream?: {ready: boolean; message?: string};
 	};
 }

@@ -294,16 +275,12 @@ export function createReadinessCheckHandler(config: HealthCheckConfig) {
 			}
 		}

-		if (services.queue !== undefined) {
-			try {
-				services.queue.engine.getStats();
-				checks.queue = {ready: true};
-			} catch (error) {
-				checks.queue = {
-					ready: false,
-					message: error instanceof Error ? error.message : 'Unknown error',
-				};
+		if (services.jsConnectionManager !== undefined) {
+			if (services.jsConnectionManager.isClosed()) {
+				checks.jetstream = {ready: false, message: 'JetStream connection is closed'};
 				allReady = false;
+			} else {
+				checks.jetstream = {ready: true};
 			}
 		}

--- a/fluxer_server/src/Routes.tsx
+++ b/fluxer_server/src/Routes.tsx
@@ -110,13 +110,6 @@ export async function mountRoutes(options: MountRoutesOptions): Promise<MountedR
 			logger.info(config.isMonolith ? 'S3 service mounted at /s3 (restricted mode)' : 'S3 service mounted at /s3');
 		}

-		if (services.queue !== undefined && !config.isMonolith) {
-			app.route('/queue', services.queue.app);
-			logger.info('Queue service mounted at /queue');
-		} else if (services.queue !== undefined) {
-			logger.info('Queue service available internally only (monolith mode)');
-		}
-
 		if (services.mediaProxy !== undefined) {
 			app.route('/media', services.mediaProxy.app);
 			logger.info(
--- a/fluxer_server/src/ServiceInitializer.tsx
+++ b/fluxer_server/src/ServiceInitializer.tsx
@@ -32,7 +32,8 @@ import {
 	setInjectedWorkerService,
 } from '@fluxer/api/src/middleware/ServiceRegistry';
 import {createTestHarnessResetHandler, registerTestHarnessReset} from '@fluxer/api/src/test/TestHarnessReset';
-import {DirectWorkerService} from '@fluxer/api/src/worker/DirectWorkerService';
+import {JetStreamWorkerQueue} from '@fluxer/api/src/worker/JetStreamWorkerQueue';
+import {WorkerService} from '@fluxer/api/src/worker/WorkerService';
 import {createAppServer} from '@fluxer/app_proxy/src/AppServer';
 import type {AppServerResult} from '@fluxer/app_proxy/src/AppServerTypes';
 import {getBuildMetadata} from '@fluxer/config/src/BuildMetadata';
@@ -43,14 +44,10 @@ import {KVClient} from '@fluxer/kv_client/src/KVClient';
 import type {Logger} from '@fluxer/logger/src/Logger';
 import type {MediaProxyAppResult} from '@fluxer/media_proxy/src/App';
 import {createMediaProxyApp} from '@fluxer/media_proxy/src/App';
-import {createQueueApp, type QueueAppResult} from '@fluxer/queue/src/App';
-import {defaultQueueConfig} from '@fluxer/queue/src/types/QueueConfig';
+import {JetStreamConnectionManager} from '@fluxer/nats/src/JetStreamConnectionManager';
 import type {S3AppResult} from '@fluxer/s3/src/App';
 import {createS3App} from '@fluxer/s3/src/App';
 import {setUser} from '@fluxer/sentry/src/Sentry';
-import {DirectQueueProvider} from '@fluxer/worker/src/providers/DirectQueueProvider';
-
-type LoggerFactory = (name: string) => Logger;

 export interface ServiceInitializationContext {
 	config: Config;
@@ -61,7 +58,7 @@ export interface ServiceInitializationContext {
 export interface InitializedServices {
 	kv?: IKVProvider;
 	s3?: S3AppResult;
-	queue?: QueueAppResult;
+	jsConnectionManager?: JetStreamConnectionManager;
 	mediaProxy?: MediaProxyAppResult;
 	admin?: AdminAppResult;
 	api?: APIAppResult;
@@ -119,41 +116,36 @@ function createS3Initializer(context: ServiceInitializationContext): ServiceInit
 	};
 }

-function createQueueInitializer(context: ServiceInitializationContext): ServiceInitializer {
+function createJetStreamInitializer(context: ServiceInitializationContext): ServiceInitializer {
 	const {config, logger} = context;
-	const baseLogger = logger.child({component: 'queue'});
-	const telemetry = createServiceTelemetry({serviceName: 'fluxer-queue', skipPaths: ['/_health']});
+	const baseLogger = logger.child({component: 'jetstream'});

-	const loggerFactory: LoggerFactory = (name: string) => baseLogger.child({module: name});
-
-	const queueApp = createQueueApp({
-		loggerFactory,
-		config: {
-			...defaultQueueConfig,
-			dataDir: requireValue(config.services.queue?.data_dir, 'services.queue.data_dir'),
-			defaultVisibilityTimeoutMs: requireValue(
-				config.services.queue?.default_visibility_timeout_ms,
-				'services.queue.default_visibility_timeout_ms',
-			),
-		},
-		metricsCollector: telemetry.metricsCollector,
-		tracing: telemetry.tracing,
+	const natsConfig = config.services.nats;
+	const connectionManager = new JetStreamConnectionManager({
+		url: natsConfig?.jetstream_url ?? 'nats://127.0.0.1:4223',
+		token: natsConfig?.auth_token || undefined,
+		name: 'fluxer-server-worker',
 	});

 	return {
-		name: 'Queue',
-		initialize: () => {
-			baseLogger.info('Queue service initialized');
-		},
-		start: async () => {
-			baseLogger.info('Starting Queue engine and cron scheduler');
-			await queueApp.start();
+		name: 'JetStream',
+		initialize: async () => {
+			await connectionManager.connect();
+			baseLogger.info('JetStream connection established');
+
+			const workerQueue = new JetStreamWorkerQueue(connectionManager);
+			await workerQueue.ensureInfrastructure();
+			baseLogger.info('JetStream stream and consumer verified');
+
+			const workerService = new WorkerService(workerQueue);
+			setInjectedWorkerService(workerService);
+			baseLogger.info('JetStream worker service injected');
 		},
 		shutdown: async () => {
-			baseLogger.info('Shutting down Queue service');
-			await queueApp.shutdown();
+			baseLogger.info('Draining JetStream connection');
+			await connectionManager.drain();
 		},
-		service: queueApp,
+		service: connectionManager,
 	};
 }

@@ -378,23 +370,10 @@ export async function initializeAllServices(context: ServiceInitializationContex
 			setInjectedS3Service(services.s3.getS3Service());
 		}

-		rootLogger.info('Initializing Queue service');
-		const queueInit = createQueueInitializer(context);
-		initializers.push(queueInit);
-		services.queue = queueInit.service as QueueAppResult;
-
-		if (services.queue) {
-			rootLogger.info('Wiring DirectWorkerService for in-process communication');
-			const directQueueProvider = new DirectQueueProvider({
-				engine: services.queue.engine,
-				cronScheduler: services.queue.cronScheduler,
-			});
-			const directWorkerService = new DirectWorkerService({
-				queueProvider: directQueueProvider,
-				logger: context.logger.child({component: 'direct-worker-service'}),
-			});
-			setInjectedWorkerService(directWorkerService);
-		}
+		rootLogger.info('Initializing JetStream worker queue');
+		const jetStreamInit = createJetStreamInitializer(context);
+		initializers.push(jetStreamInit);
+		services.jsConnectionManager = jetStreamInit.service as JetStreamConnectionManager;

 		rootLogger.info('Initializing Media Proxy service');
 		const mediaProxyInit = await createMediaProxyInitializer(context, {publicOnly: context.config.isMonolith});
@@ -441,7 +420,6 @@ export async function initializeAllServices(context: ServiceInitializationContex
 			registerTestHarnessReset(
 				createTestHarnessResetHandler({
 					kvProvider: services.kv,
-					queueEngine: services.queue?.engine,
 					s3Service: services.s3?.getS3Service(),
 				}),
 			);
--- a/fluxer_server/src/index.tsx
+++ b/fluxer_server/src/index.tsx
@@ -25,12 +25,14 @@ import {mountRoutes} from '@app/Routes';
 import {createGatewayProcessManager, type GatewayProcessManager} from '@app/utils/GatewayProcessManager';
 import {createGatewayProxy} from '@app/utils/GatewayProxy';
 import {getSnowflakeService} from '@fluxer/api/src/middleware/ServiceRegistry';
+import {CronScheduler} from '@fluxer/api/src/worker/CronScheduler';
+import {JetStreamWorkerQueue} from '@fluxer/api/src/worker/JetStreamWorkerQueue';
+import {setWorkerDependencies} from '@fluxer/api/src/worker/WorkerContext';
 import {initializeWorkerDependencies} from '@fluxer/api/src/worker/WorkerDependencies';
+import {WorkerRunner} from '@fluxer/api/src/worker/WorkerRunner';
 import {workerTasks} from '@fluxer/api/src/worker/WorkerTaskRegistry';
 import {createServerWithUpgrade} from '@fluxer/hono/src/Server';
 import type {BaseHonoEnv} from '@fluxer/hono_types/src/HonoTypes';
-import {DirectQueueProvider} from '@fluxer/worker/src/providers/DirectQueueProvider';
-import {createWorker, type WorkerResult} from '@fluxer/worker/src/runtime/WorkerFactory';
 import type {Hono} from 'hono';

 export interface FluxerServerOptions {
@@ -55,7 +57,8 @@ export async function createFluxerServer(options: FluxerServerOptions = {}): Pro
 	});

 	let server: Server | null = null;
-	let worker: WorkerResult | null = null;
+	let workerRunner: WorkerRunner | null = null;
+	let cronScheduler: CronScheduler | null = null;
 	let gatewayManager: GatewayProcessManager | null = null;
 	let isShuttingDown = false;

@@ -66,12 +69,11 @@ export async function createFluxerServer(options: FluxerServerOptions = {}): Pro
 				port: config.port,
 				env: config.env,
 				database: config.database.backend,
-				workerEnabled: config.services.queue !== undefined,
 			},
 			'Starting Fluxer Server',
 		);

-		Logger.info('Starting background services (queue engine, cron scheduler)');
+		Logger.info('Starting background services');
 		await mounted.start();

 		const shouldStartGatewayProcess =
@@ -82,40 +84,41 @@ export async function createFluxerServer(options: FluxerServerOptions = {}): Pro
 			await gatewayManager.start();
 		}

-		if (config.services.queue !== undefined) {
+		if (mounted.services.jsConnectionManager) {
 			const workerLogger = createComponentLogger('worker');

-			let queueProvider: DirectQueueProvider | undefined;
-			if (mounted.services.queue) {
-				workerLogger.info('Creating DirectQueueProvider for in-process communication');
-				queueProvider = new DirectQueueProvider({
-					engine: mounted.services.queue.engine,
-					cronScheduler: mounted.services.queue.cronScheduler,
-				});
-			}
-
 			workerLogger.info('Initializing worker dependencies');
 			const snowflakeService = getSnowflakeService();
 			await snowflakeService.initialize();
-			const workerDependencies = await initializeWorkerDependencies(snowflakeService);
+			const workerDeps = await initializeWorkerDependencies(snowflakeService);
+			setWorkerDependencies(workerDeps);

-			worker = createWorker({
-				queue: {
-					queueBaseUrl: config.internal.queue,
-					queueProvider,
-				},
-				runtime: {
-					concurrency: config.services.queue.concurrency ?? 1,
-				},
-				logger: workerLogger,
-				dependencies: workerDependencies,
+			const workerQueue = new JetStreamWorkerQueue(mounted.services.jsConnectionManager);
+			const concurrency = 5;
+
+			cronScheduler = new CronScheduler(workerQueue, workerLogger);
+			cronScheduler.upsert('processAssetDeletionQueue', 'processAssetDeletionQueue', {}, '0 */5 * * * *');
+			cronScheduler.upsert('processCloudflarePurgeQueue', 'processCloudflarePurgeQueue', {}, '0 */2 * * * *');
+			cronScheduler.upsert(
+				'processPendingBulkMessageDeletions',
+				'processPendingBulkMessageDeletions',
+				{},
+				'0 */10 * * * *',
+			);
+			cronScheduler.upsert('processInactivityDeletions', 'processInactivityDeletions', {}, '0 0 */6 * * *');
+			cronScheduler.upsert('expireAttachments', 'expireAttachments', {}, '0 0 */12 * * *');
+			cronScheduler.upsert('syncDiscoveryIndex', 'syncDiscoveryIndex', {}, '0 */15 * * * *');
+			cronScheduler.start();
+			workerLogger.info('Cron scheduler started');
+
+			workerRunner = new WorkerRunner({
+				tasks: workerTasks,
+				queue: workerQueue,
+				concurrency,
 			});

-			workerLogger.info({taskCount: Object.keys(workerTasks).length}, 'Registering worker tasks');
-			worker.registerTasks(workerTasks);
-
-			workerLogger.info({concurrency: config.services.queue.concurrency ?? 1}, 'Starting embedded worker');
-			await worker.start();
+			workerLogger.info({taskCount: Object.keys(workerTasks).length, concurrency}, 'Starting embedded worker');
+			await workerRunner.start();
 		}

 		const gatewayProxy = createGatewayProxy();
@@ -154,9 +157,15 @@ export async function createFluxerServer(options: FluxerServerOptions = {}): Pro
 			{
 				name: 'Worker',
 				fn: async () => {
-					if (worker !== null) {
+					if (cronScheduler !== null) {
+						Logger.info('Stopping cron scheduler');
+						cronScheduler.stop();
+						cronScheduler = null;
+					}
+					if (workerRunner !== null) {
 						Logger.info('Stopping embedded worker');
-						await worker.shutdown();
+						await workerRunner.stop();
+						workerRunner = null;
 					}
 				},
 			},
--- a/fluxer_server/src/utils/GatewayProcessManager.tsx
+++ b/fluxer_server/src/utils/GatewayProcessManager.tsx
@@ -130,7 +130,6 @@ export function createGatewayProcessManager(): GatewayProcessManager {
 			...process.env,
 			FLUXER_GATEWAY_HOST: Config.services.gateway.port ? '0.0.0.0' : '127.0.0.1',
 			FLUXER_GATEWAY_PORT: Config.services.gateway.port.toString(),
-			FLUXER_GATEWAY_API_HOST: Config.services.gateway.api_host,
 			FLUXER_GATEWAY_NODE_FLAG: '-name',
 			FLUXER_GATEWAY_NODE_NAME: currentNodeName,
 			ERL_DIST_PORT: GATEWAY_DIST_PORT.toString(),