import { createClient } from 'redis'; import type { RedisClientType, RedisClientOptions } from 'redis'; import { v4 as uuidv4 } from 'uuid'; import logger from '@alga-psa/core/logger'; import { getSecret } from '../../core/getSecret'; import { WorkflowEventBase, RedisStreamMessage, parseStreamEvent } from './workflowEventSchema'; /** * Configuration options for the Redis Stream client */ export interface RedisStreamConfig { url: string; password?: string; streamPrefix: string; consumerGroup: string; maxStreamLength: number; blockingTimeout: number; claimTimeout: number; batchSize: number; maxRetries: number; deadLetterQueueSuffix: string; reconnectStrategy: { retries: number; initialDelay: number; maxDelay: number; }; } /** * Default configuration for Redis Stream client */ const DEFAULT_CONFIG: RedisStreamConfig = { url: `redis://${process.env.REDIS_HOST || 'localhost'}:${process.env.REDIS_PORT || '6379'}`, streamPrefix: 'workflow:events:', consumerGroup: 'workflow-processors', maxStreamLength: 1000, blockingTimeout: 300000, // 5 minutes in ms claimTimeout: 30000, // ms batchSize: 10, maxRetries: 3, // Maximum number of retries before moving to DLQ deadLetterQueueSuffix: 'dlq', // Suffix for dead letter queue streams reconnectStrategy: { retries: 10, initialDelay: 100, // ms maxDelay: 3000, // ms }, }; /** * Options for consuming messages from Redis Streams */ export interface ConsumeOptions { count?: number; block?: number; noAck?: boolean; } /** * Redis Stream for workflow events * Provides methods for publishing events to Redis Streams and consuming events from Redis Streams */ export class RedisStreamClient { private static createdConsumerGroups: Set = new Set(); private client: any | null = null; private config: RedisStreamConfig; private consumerId: string; private isConnected: boolean = false; private isConsumerRunning: boolean = false; private consumerHandlers: Map Promise> = new Map(); /** * Create a new Redis Stream client * @param config Configuration options for the Redis Stream client */ constructor(config: Partial = {}) { this.config = { ...DEFAULT_CONFIG, ...config }; this.consumerId = `consumer-${process.pid}-${uuidv4().substring(0, 8)}`; } /** * Initialize the Redis client and connect to Redis */ public async initialize(): Promise { if (this.client && this.isConnected) { return; } try { const password = await getSecret('redis_password', 'REDIS_PASSWORD'); if (!password) { logger.warn('[RedisStreamClient] No Redis password configured - this is not recommended for production'); } const options: RedisClientOptions = { url: this.config.url, password, socket: { reconnectStrategy: (retries) => { if (retries > this.config.reconnectStrategy.retries) { return new Error('Max reconnection attempts reached'); } const delay = Math.min( this.config.reconnectStrategy.initialDelay * Math.pow(2, retries), this.config.reconnectStrategy.maxDelay ); logger.info(`[RedisStreamClient] Reconnecting in ${delay}ms (attempt ${retries})`); return delay; } } }; this.client = createClient(options); this.client.on('error', (err: Error) => { logger.error('[RedisStreamClient] Redis Client Error:', err); this.isConnected = false; }); this.client.on('connect', () => { logger.info('[RedisStreamClient] Redis Client Connected'); this.isConnected = true; }); this.client.on('reconnecting', () => { logger.info('[RedisStreamClient] Redis Client Reconnecting'); this.isConnected = false; }); this.client.on('end', () => { logger.info('[RedisStreamClient] Redis Client Connection Closed'); this.isConnected = false; }); await this.client.connect(); this.isConnected = true; logger.info('[RedisStreamClient] Redis Stream Client initialized'); } catch (error) { logger.error('[RedisStreamClient] Failed to initialize Redis client:', error); throw error; } } /** * Get the Redis client, initializing it if necessary * @returns The Redis client */ private async getClient(): Promise { if (!this.client || !this.isConnected) { await this.initialize(); } return this.client!; } /** * Get the full stream name for a workflow execution * @param executionId The workflow execution ID * @returns The full stream name */ private getStreamName(executionId: string): string { return `${this.config.streamPrefix}${executionId}`; } /** * Ensure a stream and consumer group exist * @param streamName The name of the stream */ private async ensureStreamAndGroup(streamName: string): Promise { const cacheKey = `${streamName}::${this.config.consumerGroup}`; // Check if we've already created this consumer group for this stream if (RedisStreamClient.createdConsumerGroups.has(cacheKey)) { // logger.debug(`[RedisStreamClient] Consumer group already ensured for stream: ${streamName}`); return; } try { const client = await this.getClient(); await client.xGroupCreate(streamName, this.config.consumerGroup, '0', { MKSTREAM: true }); logger.info(`[RedisStreamClient] Created consumer group ${this.config.consumerGroup} for stream: ${streamName}`); // Add to the set of created consumer groups RedisStreamClient.createdConsumerGroups.add(cacheKey); } catch (err: any) { if (err.message.includes('BUSYGROUP')) { logger.info(`[RedisStreamClient] Consumer group ${this.config.consumerGroup} already exists for stream: ${streamName}`); // Add to the set of created consumer groups even if it already existed RedisStreamClient.createdConsumerGroups.add(cacheKey); } else { logger.error(`[RedisStreamClient] Error creating consumer group ${this.config.consumerGroup} for stream ${streamName}:`, err); throw err; } } } /** * Publish a workflow event to Redis Streams * @param event The workflow event to publish * @returns The ID of the message in the stream */ public async publishEvent(event: WorkflowEventBase): Promise { try { const client = await this.getClient(); // All events will be published to the global stream as per user request. // The original event.execution_id is still part of the event payload for routing by consumers. const streamName = this.getStreamName('global'); // Publish to global stream // Ensure the stream and consumer group exist for the global stream await this.ensureStreamAndGroup(streamName); // Add the event to the stream // Store key fields as top-level entries in the Redis message for easier routing/filtering, // and the payload as a JSON string. const messageFields: { [key: string]: string } = { event_id: event.event_id, execution_id: event.execution_id || '', event_name: event.event_name, event_type: event.event_type, tenant: event.tenant, timestamp: event.timestamp, // This is already a string from Zod schema user_id: event.user_id || '', from_state: event.from_state || '', to_state: event.to_state || '', payload_json: JSON.stringify(event.payload || {}) }; const messageId = await client.xAdd( streamName, // Use the global stream name '*', // Auto-generate ID messageFields, { TRIM: { strategy: 'MAXLEN', threshold: this.config.maxStreamLength, strategyModifier: '~' // Approximate trimming for better performance } } ); logger.info(`[RedisStreamClient] Published event to stream ${streamName} with ID ${messageId}`, { eventId: event.event_id, eventName: event.event_name, executionId: event.execution_id }); return messageId; } catch (error) { logger.error('[RedisStreamClient] Failed to publish event:', error); throw error; } } /** * Publish a raw message to a specific stream (for notification channels) * @param streamName The stream name to publish to * @param messageFields The message fields to publish * @returns The ID of the message in the stream */ public async publishToStream( streamName: string, messageFields: Record ): Promise { try { const client = await this.getClient(); const messageId = await client.xAdd( streamName, '*', // Auto-generate ID messageFields, { TRIM: { strategy: 'MAXLEN', threshold: this.config.maxStreamLength, strategyModifier: '~' } } ); logger.debug(`[RedisStreamClient] Published message to stream ${streamName} with ID ${messageId}`); return messageId; } catch (error) { logger.error(`[RedisStreamClient] Failed to publish to stream ${streamName}:`, error); throw error; } } /** * Read new messages from a stream using a consumer group * @param executionId The workflow execution ID * @param options Options for consuming messages * @returns Array of stream messages */ public async readGroupMessages( executionId: string, options: ConsumeOptions = {} ): Promise { try { const client = await this.getClient(); const streamName = this.getStreamName(executionId); // Ensure the stream and consumer group exist await this.ensureStreamAndGroup(streamName); const count = options.count || this.config.batchSize; const block = options.block || this.config.blockingTimeout; logger.debug(`[RedisStreamClient] Reading from stream ${streamName} with consumer ${this.consumerId} in group ${this.config.consumerGroup}`); // First, check if there are any pending messages for this consumer const pendingInfo = await client.xPending( streamName, this.config.consumerGroup ); // If there are pending messages, read those first let streamEntries; if (pendingInfo && pendingInfo.pending > 0) { // Read pending messages streamEntries = await client.xReadGroup( this.config.consumerGroup, this.consumerId, [{ key: streamName, id: '0' }], // '0' means read pending messages { COUNT: count, BLOCK: block } ); } else { // No pending messages, read new messages streamEntries = await client.xReadGroup( this.config.consumerGroup, this.consumerId, [{ key: streamName, id: '>' }], // '>' means only new messages { COUNT: count, BLOCK: block } ); } if (!streamEntries || streamEntries.length === 0) { logger.debug(`[RedisStreamClient] No messages found in stream ${streamName}`); return []; } logger.info(`[RedisStreamClient] Successfully read ${streamEntries.length} entries from stream ${streamName}`); // Extract messages from the stream entries const resultMessages: RedisStreamMessage[] = []; for (const { messages: streamMessages } of streamEntries) { for (const message of streamMessages) { resultMessages.push({ id: message.id, message: message.message }); } } return resultMessages; } catch (error) { logger.error(`[RedisStreamClient] Failed to read messages from stream for execution ${executionId}:`, error); throw error; } } /** * Acknowledge a message in a stream * @param executionId The workflow execution ID * @param messageId The ID of the message to acknowledge */ public async acknowledgeMessage(executionId: string, messageId: string): Promise { try { const client = await this.getClient(); const streamName = this.getStreamName(executionId); await client.xAck(streamName, this.config.consumerGroup, messageId); logger.debug(`[RedisStreamClient] Acknowledged message ${messageId} in stream ${streamName}`); } catch (error) { logger.error(`[RedisStreamClient] Failed to acknowledge message ${messageId} in stream for execution ${executionId}:`, error); throw error; } } /** * Claim pending messages that have been idle for too long * @param executionId The workflow execution ID * @returns Array of claimed messages */ public async claimPendingMessages(executionId: string): Promise { try { const client = await this.getClient(); const streamName = this.getStreamName(executionId); // Get pending messages information const pendingInfo = await client.xPending( streamName, this.config.consumerGroup ); if (pendingInfo.pending === 0) { return []; } // Use XAUTOCLAIM to automatically claim messages that have been idle for too long // This is more efficient than manually filtering and claiming const { messages } = await client.xAutoClaim( streamName, this.config.consumerGroup, this.consumerId, this.config.claimTimeout, '0-0', // Start with the oldest message { COUNT: Math.min(this.config.batchSize, 10) // Limit batch size to prevent memory issues } ); if (!messages || messages.length === 0) { return []; } // Convert to RedisStreamMessage format return messages .filter(message => message !== null) .map(message => ({ id: message!.id, message: message!.message })); } catch (error: any) { // If XAUTOCLAIM is not supported (Redis < 6.2), fall back to the old method if (error.message && error.message.includes('ERR unknown command')) { logger.warn(`[RedisStreamClient] XAUTOCLAIM not supported, falling back to XCLAIM`); return this.claimPendingMessagesLegacy(executionId); } logger.error(`[RedisStreamClient] Failed to claim pending messages for execution ${executionId}:`, error); throw error; } } /** * Legacy method to claim pending messages for Redis < 6.2 * @param executionId The workflow execution ID * @returns Array of claimed messages */ private async claimPendingMessagesLegacy(executionId: string): Promise { try { const client = await this.getClient(); const streamName = this.getStreamName(executionId); // Get detailed information about pending messages const pendingMessages = await client.xPendingRange( streamName, this.config.consumerGroup, '-', // Start with the oldest message '+', // End with the newest message Math.min(this.config.batchSize, 10) // Limit batch size to prevent memory issues ); if (!pendingMessages || pendingMessages.length === 0) { return []; } // Filter messages that have been idle for too long const now = Date.now(); const claimIds = pendingMessages .filter(msg => (now - msg.millisecondsSinceLastDelivery) > this.config.claimTimeout) .map(msg => msg.id); if (claimIds.length === 0) { return []; } // Claim the messages const claimedMessages = await client.xClaim( streamName, this.config.consumerGroup, this.consumerId, this.config.claimTimeout, claimIds ); if (!claimedMessages || claimedMessages.length === 0) { return []; } // Convert to RedisStreamMessage format return claimedMessages .filter(message => message !== null) .map(message => ({ id: message!.id, message: message!.message })); } catch (error) { logger.error(`[RedisStreamClient] Failed to claim pending messages (legacy) for execution ${executionId}:`, error); throw error; } } /** * Register a handler for consuming events from a specific execution * @param executionId The workflow execution ID * @param handler The handler function to process events */ public registerConsumer( executionId: string, handler: (event: WorkflowEventBase) => Promise ): void { this.consumerHandlers.set(executionId, handler); const streamName = this.getStreamName(executionId); logger.info(`[RedisStreamClient] Registered consumer for execution ${executionId}, will read from stream: ${streamName}`); // Start the consumer if it's not already running if (!this.isConsumerRunning) { this.startConsumer(); } } /** * Unregister a consumer handler for a specific execution * @param executionId The workflow execution ID */ public unregisterConsumer(executionId: string): void { this.consumerHandlers.delete(executionId); logger.info(`[RedisStreamClient] Unregistered consumer for execution ${executionId}`); } /** * Start the consumer loop to process events from all registered executions */ private async startConsumer(): Promise { if (this.isConsumerRunning) { return; } this.isConsumerRunning = true; logger.info('[RedisStreamClient] Starting consumer loop'); const processEvents = async () => { if (!this.isConsumerRunning) { return; } try { // Process each registered execution for (const [executionId, handler] of this.consumerHandlers.entries()) { try { // Log which stream we're reading from const streamName = this.getStreamName(executionId); logger.debug(`[RedisStreamClient] Attempting to read from stream: ${streamName}`); // Read new messages const messages = await this.readGroupMessages(executionId); if (messages.length > 0) { logger.info(`[RedisStreamClient] Processing ${messages.length} messages from stream ${streamName}`); } // Process messages in batches to improve efficiency const processingPromises: Promise[] = []; for (const message of messages) { processingPromises.push((async () => { try { // Parse the event const event = parseStreamEvent(message); // Process the event await handler(event); // Acknowledge the message await this.acknowledgeMessage(executionId, message.id); } catch (error) { logger.error(`[RedisStreamClient] Error processing message ${message.id} for execution ${executionId}:`, error); // Get the number of times this message has been delivered try { const client = await this.getClient(); const pendingInfo = await client.xPendingRange( this.getStreamName(executionId), this.config.consumerGroup, message.id, message.id, 1 ); if (pendingInfo && pendingInfo.length > 0) { const deliveryCount = pendingInfo[0].deliveriesCounter; // If the message has been delivered too many times, move it to the DLQ if (deliveryCount >= this.config.maxRetries) { logger.warn(`[RedisStreamClient] Message ${message.id} has been delivered ${deliveryCount} times, moving to DLQ`); await this.moveToDeadLetterQueue(executionId, message.id, message, error); } else { logger.info(`[RedisStreamClient] Message ${message.id} will be retried (${deliveryCount}/${this.config.maxRetries})`); // Don't acknowledge the message so it can be retried } } } catch (pendingError) { logger.error(`[RedisStreamClient] Error checking pending info for message ${message.id}:`, pendingError); // Don't acknowledge the message so it can be retried } } })()); } // Wait for all messages to be processed if (processingPromises.length > 0) { await Promise.all(processingPromises); } // Claim and process pending messages const claimedMessages = await this.claimPendingMessages(executionId); if (claimedMessages.length > 0) { logger.debug(`[RedisStreamClient] Processing ${claimedMessages.length} claimed messages for execution ${executionId}`); } // Process claimed messages in batches const claimedProcessingPromises: Promise[] = []; for (const message of claimedMessages) { claimedProcessingPromises.push((async () => { try { // Parse the event const event = parseStreamEvent(message); // Process the event await handler(event); // Acknowledge the message await this.acknowledgeMessage(executionId, message.id); } catch (error) { logger.error(`[RedisStreamClient] Error processing claimed message ${message.id} for execution ${executionId}:`, error); // Get the number of times this message has been delivered try { const client = await this.getClient(); const pendingInfo = await client.xPendingRange( this.getStreamName(executionId), this.config.consumerGroup, message.id, message.id, 1 ); if (pendingInfo && pendingInfo.length > 0) { const deliveryCount = pendingInfo[0].deliveriesCounter; // If the message has been delivered too many times, move it to the DLQ if (deliveryCount >= this.config.maxRetries) { logger.warn(`[RedisStreamClient] Claimed message ${message.id} has been delivered ${deliveryCount} times, moving to DLQ`); await this.moveToDeadLetterQueue(executionId, message.id, message, error); } else { logger.info(`[RedisStreamClient] Claimed message ${message.id} will be retried (${deliveryCount}/${this.config.maxRetries})`); // Don't acknowledge the message so it can be retried } } } catch (pendingError) { logger.error(`[RedisStreamClient] Error checking pending info for claimed message ${message.id}:`, pendingError); // Don't acknowledge the message so it can be retried } } })()); } // Wait for all claimed messages to be processed if (claimedProcessingPromises.length > 0) { await Promise.all(claimedProcessingPromises); } } catch (error) { logger.error(`[RedisStreamClient] Error processing execution ${executionId}:`, error); // Continue with the next execution } } // Continue the loop with a small delay to prevent excessive CPU usage setTimeout(processEvents, 100); } catch (error) { logger.error('[RedisStreamClient] Error in consumer loop:', error); // Continue the loop after a delay setTimeout(processEvents, 1000); } }; // Start the consumer loop processEvents(); } /** * Stop the consumer loop */ public stopConsumer(): void { this.isConsumerRunning = false; logger.info('[RedisStreamClient] Stopping consumer loop'); } /** * Acquire a distributed lock * * @param key Lock key * @param owner Identifier of the lock owner * @param ttlMs Time-to-live in milliseconds * @returns True if lock was acquired, false otherwise */ public async acquireLock(key: string, owner: string, ttlMs: number): Promise { try { const client = await this.getClient(); // Use Redis SET with NX option (only set if key doesn't exist) const result = await client.set( `lock:${key}`, owner, { PX: ttlMs, // Expire in milliseconds NX: true // Only set if key doesn't exist } ); return result === 'OK'; } catch (error) { logger.error(`[RedisStreamClient] Error acquiring lock ${key}:`, error); throw error; } } /** * Release a distributed lock * * @param key Lock key * @param owner Identifier of the lock owner * @returns True if lock was released, false if lock doesn't exist or is owned by someone else */ public async releaseLock(key: string, owner: string): Promise { try { const client = await this.getClient(); // Use Lua script to ensure we only delete the lock if we own it const script = ` if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end `; const result = await client.eval( script, { keys: [`lock:${key}`], arguments: [owner] } ); return result === 1; } catch (error) { logger.error(`[RedisStreamClient] Error releasing lock ${key}:`, error); throw error; } } /** * Extend a lock's TTL * * @param key Lock key * @param owner Identifier of the lock owner * @param ttlMs New time-to-live in milliseconds * @returns True if lock was extended, false if lock doesn't exist or is owned by someone else */ public async extendLock(key: string, owner: string, ttlMs: number): Promise { try { const client = await this.getClient(); // Use Lua script to ensure we only extend the lock if we own it const script = ` if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end `; const result = await client.eval( script, { keys: [`lock:${key}`], arguments: [owner, ttlMs.toString()] } ); return result === 1; } catch (error) { logger.error(`[RedisStreamClient] Error extending lock ${key}:`, error); throw error; } } /** * List messages in the dead letter queue for a specific execution * @param executionId The workflow execution ID * @param count Maximum number of messages to return * @returns Array of messages in the DLQ */ public async listDeadLetterQueueMessages( executionId: string, count: number = 100 ): Promise { try { const client = await this.getClient(); const streamName = `${this.getStreamName(executionId)}:${this.config.deadLetterQueueSuffix}`; // Read messages from the DLQ const messages = await client.xRange( streamName, '-', // Start with the oldest message '+', // End with the newest message { COUNT: count } ); return messages.map(msg => ({ id: msg.id, ...msg.message })); } catch (error) { logger.error(`[RedisStreamClient] Failed to list DLQ messages for execution ${executionId}:`, error); return []; } } /** * Reprocess a message from the dead letter queue * @param executionId The workflow execution ID * @param dlqMessageId The ID of the message in the DLQ * @returns True if the message was successfully reprocessed, false otherwise */ public async reprocessDeadLetterQueueMessage( executionId: string, dlqMessageId: string ): Promise { try { const client = await this.getClient(); const dlqStreamName = `${this.getStreamName(executionId)}:${this.config.deadLetterQueueSuffix}`; const targetStreamName = this.getStreamName(executionId); // Get the message from the DLQ const messages = await client.xRange( dlqStreamName, dlqMessageId, dlqMessageId ); if (!messages || messages.length === 0) { logger.error(`[RedisStreamClient] Message ${dlqMessageId} not found in DLQ for execution ${executionId}`); return false; } const dlqMessage = messages[0]; // Extract the original message let originalMessage; try { originalMessage = JSON.parse(dlqMessage.message.original_message); } catch (parseError) { logger.error(`[RedisStreamClient] Failed to parse original message from DLQ message ${dlqMessageId}:`, parseError); return false; } // Add the message back to the original stream // Ensure it's formatted correctly, like in publishEvent const messageFields: { [key: string]: string } = { event_id: originalMessage.event_id, execution_id: originalMessage.execution_id || '', event_name: originalMessage.event_name, event_type: originalMessage.event_type, tenant: originalMessage.tenant, timestamp: originalMessage.timestamp, // Assuming originalMessage.timestamp is already string user_id: originalMessage.user_id || '', from_state: originalMessage.from_state || '', to_state: originalMessage.to_state || '', payload_json: JSON.stringify(originalMessage.payload || {}) }; const messageId = await client.xAdd( targetStreamName, '*', // Auto-generate ID messageFields, { TRIM: { // Optional: Apply trimming similar to publishEvent if desired strategy: 'MAXLEN', threshold: this.config.maxStreamLength, strategyModifier: '~' } } ); // Delete the message from the DLQ await client.xDel(dlqStreamName, dlqMessageId); logger.info(`[RedisStreamClient] Reprocessed message ${dlqMessageId} from DLQ to stream ${targetStreamName} with ID ${messageId}`); return true; } catch (error) { logger.error(`[RedisStreamClient] Failed to reprocess message ${dlqMessageId} from DLQ for execution ${executionId}:`, error); return false; } } /** * Move a message to the dead letter queue * @param executionId The workflow execution ID * @param messageId The ID of the message to move * @param error The error that caused the message to be moved to DLQ */ public async moveToDeadLetterQueue( executionId: string, messageId: string, message: RedisStreamMessage, error: any ): Promise { try { const client = await this.getClient(); const sourceStreamName = this.getStreamName(executionId); const dlqStreamName = `${sourceStreamName}:${this.config.deadLetterQueueSuffix}`; // Add metadata about the error const dlqMessage: Record = { original_message: JSON.stringify(message.message), error_message: error instanceof Error ? error.message : String(error), error_stack: error instanceof Error && error.stack ? error.stack : 'No stack trace', source_stream: sourceStreamName, original_id: messageId, moved_at: new Date().toISOString() }; // Add the message to the DLQ const dlqMessageId = await client.xAdd( dlqStreamName, '*', // Auto-generate ID dlqMessage ); // Acknowledge the original message to remove it from pending await this.acknowledgeMessage(executionId, messageId); logger.warn(`[RedisStreamClient] Moved message ${messageId} to DLQ ${dlqStreamName} with ID ${dlqMessageId}`, { executionId, error: error instanceof Error ? error.message : String(error) }); } catch (dlqError) { logger.error(`[RedisStreamClient] Failed to move message ${messageId} to DLQ for execution ${executionId}:`, dlqError); // Don't throw the error, as we don't want to fail the processing of other messages } } /** * Close the Redis client connection */ public async close(): Promise { this.stopConsumer(); if (this.client && this.isConnected) { await this.client.quit(); this.client = null; this.isConnected = false; logger.info('[RedisStreamClient] Redis client closed'); } } } // Singleton instance let redisStreamClientInstance: RedisStreamClient | null = null; /** * Get the Redis Stream client instance * @param config Configuration options for the Redis Stream client * @returns The Redis Stream client instance */ export function getRedisStreamClient(config: Partial = {}): RedisStreamClient { if (!redisStreamClientInstance) { redisStreamClientInstance = new RedisStreamClient(config); } return redisStreamClientInstance; }