Email-queue routing parks workflow waits on a queue the subscription matcher never searches
Tier 2 easy typescript · cross-file · real merged PR from PostHog/posthog #54418
Ground-truth defects
With CDP_EMAIL_QUEUE_ROUTING on, a workflow's email step now routes the invocation to the new `email` cyclotron queue, the hog_function action copies that queue onto the flow invocation, and the email worker (a hogflow worker on queue `email`) sends the email and keeps running the flow. A `wait_until_condition` step after it therefore parks the job with queue_name='email', which job-queue-postgres-v2 now persists. The hogflow subscription matcher, unchanged by this diff, wakes parked waits through findParkedJobs, which only looks up `queue_name = 'hogflow'`. It never finds the wait, so a matching event does not advance the workflow and it resumes only on the wait's timeout branch.
The change under review
diff (truncated — full diff)
@@ -190,7 +190,8 @@ case "$RS_SVC" in export RUST_BACKTRACE=1 export KAFKA_HOSTS="$DEFAULT_KAFKA" export KAFKA_TOPIC=clickhouse_app_metrics2 - export DATABASE_URL="${DATABASE_URL:-postgres://posthog:posthog@${PGHOST:-db}:${PGPORT:-5432}/cyclotron}" + export DATABASE_URL="${CYCLOTRON_DATABASE_URL:-postgres://posthog:posthog@${PGHOST:-db}:${PGPORT:-5432}/cyclotron}" + export BIND_PORT=3310 # default 3303 collides with property-vals-rs in local dev export DEBUG=1 export ALLOW_INTERNAL_IPS=${ALLOW_INTERNAL_IPS:-true} ;; @@ -25,6 +25,7 @@ export const CAPABILITIES_CDP_WORKFLOWS: PluginServerCapabilities = { ...CAPABILITIES_CDP, cdpBatchHogFlow: true, cdpCyclotronWorkerHogFlow: true, + cdpCyclotronWorkerEmail: true, cdpCyclotronV2Janitor: isDevEnv(), cdpHogflowScheduler: isDevEnv(), cdpHogflowSubscriptionMatcher: isDevEnv(), @@ -130,6 +131,10 @@ export function getPluginServerCapabilities( return { cdpCyclotronWorkerHogFlowLegacyPg: true, } + case PluginServerMode.cdp_cyclotron_worker_email: + return { + cdpCyclotronWorkerEmail: true, + } case PluginServerMode.cdp_precalculated_filters: return { cdpPrecalculatedFilters: true, @@ -97,6 +97,7 @@ export interface CdpCoreServices { recipientTokensService: RecipientTokensService /** Resolved outputs shared across every CDP service/consumer. */ outputs: CdpOutputs + emailService: EmailService } export type CdpCoreServicesConfig = Pick< @@ -140,6 +141,7 @@ export type CdpCoreServicesConfig = Pick< | 'CDP_FETCH_RETRIES' | 'CDP_FETCH_BACKOFF_BASE_MS' | 'CDP_FETCH_BACKOFF_MAX_MS' + | 'CDP_EMAIL_QUEUE_ROUTING' | 'HOG_FUNCTION_MONITORING_APP_METRICS_TOPIC' | 'HOG_FUNCTION_MONITORING_APP_METRICS_PRODUCER' | 'HOG_FUNCTION_MONITORING_LOG_ENTRIES_TOPIC' @@ -393,6 +395,7 @@ export function createCdpCoreServices( fetchRetries: config.CDP_FETCH_RETRIES, fetchBackoffBaseMs: config.CDP_FETCH_BACKOFF_BASE_MS, fetchBackoffMaxMs: config.CDP_FETCH_BACKOFF_MAX_MS, + emailQueueRouting: config.CDP_EMAIL_QUEUE_ROUTING, }, { teamManager: deps.teamManager, siteUrl: config.SITE_URL }, hogInputsService, @@ -454,5 +457,6 @@ export function createCdpCoreServices( segmentDestinationExecutorService, recipientTokensService, outputs, + emailService, } } @@ -49,6 +49,11 @@ export type CdpConfig = ClickhouseConfig & { CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_KIND: CyclotronJobQueueKind CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: CyclotronJobQueueSource CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS: string + // Controls which teams route email sends to the dedicated email queue. + // Supports team IDs, percentage rollout, or both. + // Examples: '' (disabled), '123,456' (specific teams), '*:0.1' (10% of traffic), + // '123,*:0.05' (team 123 + 5% of rest), '*' (all traffic) + CDP_EMAIL_QUEUE_ROUTING: string CDP_LEGACY_EVENT_CONSUMER_GROUP_ID: string CDP_LEGACY_EVENT_CONSUMER_TOPIC: string @@ -163,6 +168,7 @@ export function getDefaultCdpConfig(): CdpConfig { CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_KIND: 'hog', CDP_CYCLOTRON_JOB_QUEUE_CONSUMER_MODE: 'kafka', CDP_CYCLOTRON_STRIP_PERSON_FROM_STATE_TEAMS: '', + CDP_EMAIL_QUEUE_ROUTING: '', CDP_LEGACY_EVENT_CONSUMER_GROUP_ID: 'clickhouse-plugin-server-async-onevent', CDP_LEGACY_EVENT_CONSUMER_TOPIC: KAFKA_EVENTS_JSON, @@ -27,6 +27,7 @@ import { HogFunctionManagerService } from '../services/managers/hog-function-man import { HogFunctionTemplateManagerService } from '../services/managers/hog-function-template-manager.service' import { PersonsManagerService } from '../services/managers/persons-manager.service' import { RecipientsManagerService } from '../services/managers/recipients-manager.service' +import { EmailService } from '../services/messaging/email.service' import { RecipientPreferencesService } from '../services/messaging/recipient-preferences.service' import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service' import { HogMaskerService } from '../services/monitoring/hog-masker.service' @@ -70,6 +71,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = Cd personsManager: PersonsManagerService recipientsManager: RecipientsManagerService + emailService: EmailService hogFunctionMonitoringService: HogFunctionMonitoringService invocationResultsService: InvocationResultsService nativeDestinationExecutorService: NativeDestinationExecutorService @@ -99,6 +101,7 @@ export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = Cd this.recipientsManager = services.recipientsManager this.recipientPreferencesService = services.recipientPreferencesService this.hogFlowExecutor = services.hogFlowExecutor + this.emailService = services.emailService this.hogFunctionMonitoringService = services.hogFunctionMonitoringService this.invocationResultsService = services.invocationResultsService this.nativeDestinationExecutorService = services.nativeDestinationExecutorService new file mode 100644 @@ -0,0 +1,14 @@ +import { PluginsServerConfig } from '~/types' + +import { JobQueue } from '../services/job-queue/job-queue.interface' +import { CdpConsumerBaseDeps } from './cdp-base.consumer' +import { CdpCyclotronWorkerHogFlow } from './cdp-cyclotron-worker-hogflow.consumer' + +export class CdpCyclotronWorkerEmail extends CdpCyclotronWorkerHogFlow { + protected override name = 'CdpCyclotronWorkerEmail' + + constructor(config: PluginsServerConfig, deps: CdpConsumerBaseDeps, jobQueue: JobQueue) { + super(config, deps, jobQueue) + this.queue = 'email' + } +} @@ -149,9 +149,17 @@ export class CdpEventsConsumer< return events } + protected async startQueueProducers(): Promise<void> { + await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()]) + } + + protected async stopQueueProducers(): Promise<void> { + await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()]) + } + public override async start(): Promise<void> { await super.start() - await Promise.all([this.hogQueue.startAsProducer(), this.hogflowQueue.startAsProducer()]) + await this.startQueueProducers() // Start consuming messages await this.kafkaConsumer.connect(async (messages) => { logger.info('🔁', `${this.name} - handling batch`, { @@ -171,7 +179,7 @@ export class CdpEventsConsumer< logger.info('💤', 'Stopping consumer...') await this.kafkaConsumer.disconnect() logger.info('💤', 'Stopping job queues...') - await Promise.all([this.hogQueue.stopProducer(), this.hogflowQueue.stopProducer()]) + await this.stopQueueProducers() // IMPORTANT: super always comes last await super.stop() logger.info('💤', 'Consumer stopped!') @@ -488,6 +488,7 @@ export function createHogTransformerService( fetchRetries: config.CDP_FETCH_RETRIES, fetchBackoffBaseMs: config.CDP_FETCH_BACKOFF_BASE_MS, fetchBackoffMaxMs: config.CDP_FETCH_BACKOFF_MAX_MS, + emailQueueRouting: config.CDP_EMAIL_QUEUE_ROUTING, }, { teamManager: deps.teamManager, siteUrl: config.SITE_URL }, hogInputsService, @@ -37,6 +37,7 @@ export const CyclotronV2RescheduleOptionsSchema = z.object({ distinctId: z.string().nullish(), personId: z.string().nullish(), actionId: z.string().nullish(), + queueName: z.string().optional(), }) export type CyclotronV2RescheduleOptions = z.infer<typeof CyclotronV2RescheduleOptionsSchema> @@ -211,6 +211,10 @@ export class CyclotronV2Worker { params.push(options.actionId ?? null) setClauses.push(`action_id = $${params.length}`) } + if (options?.queueName !== undefined) { + params.push(options.queueName) + setClauses.push(`queue_name = $${params.length}`) + } await pool.query( `UPDATE cyclotron_jobs SET ${setClauses.join(', ')} @@ -8,7 +8,8 @@ import { ACCESS_TOKEN_PLACEHOLDER } from '~/config/constants' import { FetchOptions, FetchResponse, InvalidRequestError, SecureRequestError, fetch } from '~/utils/request' import { tryCatch } from '~/utils/try-catch' -import { PluginsServerConfig } from '../../types' +import { buildIntegerMatcherWithPercentage } from '../../config/config' +import { PluginsServerConfig, ValueMatcher } from '../../types' import { parseJSON } from '../../utils/json-parse' import { logger } from '../../utils/logger' import { TeamManager } from '../../utils/team-manager' @@ -47,6 +48,7 @@ export interface HogExecutorConfig { fetchRetries: number fetchBackoffBaseMs: number fetchBackoffMaxMs: number + emailQueueRouting: string } export interface HogExecutorAsyncContext { @@ -54,6 +56,11 @@ export interface HogExecutorAsyncContext { siteUrl: string } +const cdpEmailQueuedTotal = new Counter({ + name: 'cdp_email_queued_total', + help: 'Total emails routed to the dedicated email queue', +}) + const cdpHttpRequests = new Counter({ name: 'cdp_http_requests', help: 'HTTP requests and their outcomes', @@ -180,13 +187,17 @@ export type HogExecutorExecuteAsyncOptions = HogExecutorExecuteOptions & { } export class HogExecutorService { + private emailQueueMatcher: ValueMatcher<number> + constructor( private config: HogExecutorConfig, private asyncContext: HogExecutorAsyncContext, private hogInputsService: HogInputsService, private emailService: EmailService, private recipientTokensService: RecipientTokensService - ) {} + ) { + this.emailQueueMatcher = buildIntegerMatcherWithPercentage(config.emailQueueRouting) + } async buildInputsWithGlobals( hogFunction: HogFunctionType, @@ -339,10 +350,33 @@ export class HogExecutorService { break } + // Queue-aware routing: each worker can execute some actions inline + // and routes others to a specialized queue. The email worker sends + // emails inline but routes fetches back to hogflow. The hogflow + // worker does fetches inline but routes emails to the email queue + // (when the team is configured for it via CDP_EMAIL_QUEUE_ROUTING). + // + // Future: once we add an execution time budget, the email worker + // will also handle fetches inline. The only reason to reschedule + // back to hogflow will be when overall execution time exceeds the + // budget, to avoid blocking the queue. if (queueParamsType === 'fetch') { - result = await this.executeFetch(nextInvocation, options) + if (invocation.queue === 'email') { + result = this.routeToQueue( + nextInvocation, + nextInvocation.queueMetadata?.originQueue ?? 'hogflow' + ) + } else { + result = await this.executeFetch(nextInvocation, options) + } } else if (queueParamsType === 'email') { - result = await this.emailService.executeSendEmail(nextInvocation) + // Route to the email queue only if we're not already there + // and the team is configured for it. Otherwise send inline. + if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) { + result = this.routeEmailToQueue(nextInvocation) + } else { + result = await this.emailService.executeSendEmail(nextInvocation) + } } else { throw new Error(`Unknown queue type: ${queueParamsType}`) } @@ -356,8 +390,8 @@ export class HogExecutorService { logs.push(...result.logs) metrics.push(...result.metrics) - // If we have finished _or_ something has been scheduled to run later _or_ we have reached the max async functions then we break the loop - if (result.finished || result.invocation.queueScheduledAt) { + // If we have finished _or_ something has been scheduled to run later _or_ the job was routed to a different queue then we break the loop + if (result.finished || result.invocation.queueScheduledAt || result.invocation.queue !== invocation.queue) { break } } @@ -376,6 +410,56 @@ export class HogExecutorService { return result } + /** + * Routes an email send to the dedicated email queue instead of sending inline. + * The email worker will pick this up, send via SES, and return the job to the + * original queue so the workflow can continue. + */ + private routeEmailToQueue( + invocation: CyclotronJobInvocationHogFunction + ): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> { + const result = createInvocationResult<CyclotronJobInvocationHogFunction>( + invocation, + { + queue: 'email', + queueParameters: invocation.queueParameters, + queueMetadata: { + ...invocation.queueMetadata, + originQueue: invocation.queue, + }, + }, + { finished: false } + ) + + result.metrics.push({ + team_id: invocation.teamId, + app_source_id: invocation.parentRunId ?? invocation.functionId, + instance_id: invocation.state.actionId || invocation.id, + metric_kind: 'email', + metric_name: 'email_queued', + count: 1, + }) + + cdpEmailQueuedTotal.inc() + + return result + } + + private routeToQueue( + invocation: CyclotronJobInvocationHogFunction, + targetQueue: string + ): CyclotronJobInvocationResult<CyclotronJobInvocationHogFunction> { + return createInvocationResult<CyclotronJobInvocationHogFunction>( + invocation, + { + queue: targetQueue as CyclotronJobInvocationHogFunction['queue'], + queueParameters: invocation.queueParameters, + queueMetadata: undefined, + }, + { finished: false } + ) + } + @instrumented({ key: 'hog-executor.execute', sendException: false }) async execute( invocation: CyclotronJobInvocationHogFunction, @@ -719,7 +803,6 @@ export class HogExecutorService { const maxRetries = options?.maxFetchRetries ?? this.config.fetchRetries if (canRetry && result.invocation.state.attempts < maxRetries) { await fetchResponse?.dump() - result.invocation.queue = 'hog' result.invocation.queueParameters = params result.invocation.queuePriority = invocation.queuePriority + 1 result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: backoffMs }) @@ -54,8 +54,10 @@ export class HogFunctionHandler implements ActionHandler { if (!functionResult.finished) { // Set the state of the function result on the substate of the flow for the next execution result.invocation.state.currentAction!.hogFunctionState = functionResult.invocation.state - // Also the queueParameters are required + // Preserve queue routing and parameters from the function result + result.invocation.queue = functionResult.invocation.queue result.invocation.queueParameters = functionResult.invocation.queueParameters + result.invocation.queueMetadata = functionResult.invocation.queueMetadata return { scheduledAt: functionResult.invocation.queueScheduledAt ?? DateTime.now(), } @@ -206,6 +206,7 @@ export class CyclotronJobQueuePostgresV2 implements JobQueue { distinctId: extractDistinctId(result.invocation), personId: extractPersonId(result.invocation), actionId: extractActionId(result.invocation), + queueName: result.invocation.queue, }) } }) @@ -112,8 +112,8 @@ export class EmailService { result.finished = true } - // Finally we create the response object as the VM expects - result.invocation.state.vmState!.stack.push({ + // Push the response to the VM stack if running inline (not from the email queue) + result.invocation.state.vmState?.stack.push({ success, }) @@ -222,7 +222,7 @@ export class EmailService { FeedbackForwardingEmailAddress: from.email, } - const isTransactionalEmail = result.invocation.hogFunction.metadata?.message_category_type === 'transactional' + const isTransactionalEmail = result.invocation.hogFunction?.metadata?.message_category_type === 'transactional' // Automatically add unsubscribe headers for non-transactional emails if (sendEmailParams.Content?.Simple && !isTransactionalEmail) { sendEmailParams.Content.Simple.Headers = this.generateUnsubscribeHeaders({ @@ -229,6 +229,7 @@ export type MinimalAppMetric = { | 'fetch' | 'billable_invocation' | 'dropped' + | 'email_queued' | 'email_sent' | 'email_delivered' | 'email_failed' @@ -253,7 +254,7 @@ export interface HogFunctionTiming { } // IMPORTANT: All queue names should be lowercase and only [A-Z0-9] characters are allowed. -export const CYCLOTRON_INVOCATION_JOB_QUEUES = ['hog', 'hogoverflow', 'hogflow'] as const +export const CYCLOTRON_INVOCATION_JOB_QUEUES = ['hog', 'hogoverflow', 'hogflow', 'email'] as const export type CyclotronJobQueueKind = (typeof CYCLOTRON_INVOCATION_JOB_QUEUES)[number] export const CYCLOTRON_JOB_QUEUE_SOURCES = ['postgres', 'postgres-v2', 'kafka'] as const @@ -27,6 +27,7 @@ export enum PluginServerMode { cdp_cohort_membership = 'cdp-cohort-membership', cdp_cyclotron_worker_hogflow = 'cdp-cyclotron-worker-hogflow', cdp_cyclotron_worker_hogflow_legacy_pg = 'cdp-cyclotron-worker-hogflow-legacy-pg', + cdp_cyclotron_worker_email = 'cdp-cyclotron-worker-email', cdp_api = 'cdp-api', cdp_legacy_on_event = 'cdp-legacy-on-event', evaluation_scheduler = 'evaluation-scheduler', @@ -142,6 +142,50 @@ export function buildIntegerMatcher(config: string | undefined, allowStar: boole } } +/** + * Builds a matcher that supports team IDs and/or percentage-based rollout. + * + * Formats: + * '' → no match + * '*' → match all + * '123,456' → only teams 123 and 456 + * '*:0.1' → 10% of all traffic (random per call) + * '123,*:0.05' → team 123 always + 5% of all other teams + */ +export function buildIntegerMatcherWithPercentage(config: string | undefined): ValueMatcher<number> { + if (!config || config.trim().length === 0) { + return () => false + } + if (config.trim() === '*') { + return () => true + } + + const parts = config.split(',').map((s) => s.trim()) + const teamIds = new Set<number>() + let percentage = 0 + + for (const part of parts) { + if (part.startsWith('*:')) { + percentage = parseFloat(part.slice(2)) + } else { + const num = parseInt(part)
Leaderboard for this case
easy · 7 of 9 bots fully caught it (78% solve rate) · 78% avg coverage.
| # | Bot | Result | Caught | Coverage | Findings | False alarms |
|---|---|---|---|---|---|---|
| 1 | Claude Code (Opus 5.5) | solved | 1/1 | 100.0% | 3 | 0 |
| 2 | cursor | solved | 1/1 | 100.0% | 2 | 0 |
| 3 | Drape (Deep Review) | solved | 1/1 | 100.0% | 5 | 0 |
| 4 | Drape (Review) | solved | 1/1 | 100.0% | 5 | 0 |
| 5 | Mjolnir (Opus 5.5) | solved | 1/1 | 100.0% | 5 | 0 |
| 6 | Mjolnir (Sonnet 5.5) | solved | 1/1 | 100.0% | 4 | 0 |
| 7 | qodo | solved | 1/1 | 100.0% | 4 | 0 |
| 8 | Cubic | miss | 0/1 | 0.0% | 3 | 0 |
| 9 | greptile | miss | 0/1 | 0.0% | 2 | 0 |
Bot results
Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
The routing sits in the generic executeWithAsyncFunctions. That method also runs plain hog destinations on the Kafka 'hog' queue, and email.template.ts calls sendEmail. For a matched team, the job's queue becomes 'email' and the Kafka queue republishes it to `cdp_cyclotron_${msg.queue}`. Nothing consumes that topic: the email worker reads only from postgres-v2, and it is a HogFlow worker. [grounded: diff] [consensus 3/3: valid=3]
Per-call randomness is the documented design ('random per call'), and it causes no wrong result because routed emails are still sent by the email worker. A NaN percentage just disables routing. The finding asks for a design change (hashing the team ID, stricter parsing) rather than showing a malfunction. [grounded: diff] [consensus 3/4: minor=3, valid=1]
raw comments (3)
Email routing runs inside the generic `HogExecutorService.executeWithAsyncFunctions`, so it applies to every hog function, not only hogflow actions. Plain hog destinations also call `sendEmail` (for example `templates/_destinations/email/email.template.ts`). These run on the `hog` worker, which uses the Kafka job queue. For a team matched by `CDP_EMAIL_QUEUE_ROUTING`, the job's queue is changed to `email`, so the Kafka queue re-publishes it to the `cdp_cyclotron_email` topic. Nothing consumes that topic, because the email worker only reads from postgres-v2. Even if the job did reach the email worker, `CdpCyclotronWorkerEmail` is a HogFlow worker, so `loadHogFlows` would not find the hog function ID and would fail the job. Either way the email is silently dropped. The CDP API test-invocation path has a similar problem: it gets back an unfinished, re-queued result instead of a sent email. The routing should be limited to hogflow invocations (queue `hogflow`) that use the postgres-v2 queue.
The design described in the comments ("the email worker will ... send via SES, and return the job to the original queue so the workflow can continue") is not implemented. After the email worker sends the email, the result keeps `queue: 'email'` and `queueMetadata.originQueue`. Because `CdpCyclotronWorkerEmail` is a full HogFlow worker, it carries on with the rest of the workflow: delays, wait-until conditions, branches and later emails. All of these get rescheduled onto the `email` queue. The job only goes back to the origin queue if a later step does a fetch. In practice whole workflows, including long delays, end up on the dedicated email queue and its worker, which defeats the isolation the queue is meant to provide. When the email is sent from the `email` queue, the invocation should be rescheduled to `originQueue`.
The percentage rollout uses `Math.random()` on every call, so the same team and workflow can be routed differently from one email to the next. Because it isn't based on the team, a gradual rollout like `*:0.1` can't be used to keep specific teams consistently on or off the new path, which makes debugging and comparison hard. Hashing the team ID would make the routing deterministic. The parsing also accepts malformed values silently: `parseFloat` returning `NaN`, or `parseInt('12abc')` returning 12.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✗ | — |
`sendEmail` is registered as a global async function, so standalone hog functions on the 'hog' queue can call it too. The routing check only tests `queue !== 'email'` plus the team matcher, so these invocations get moved to the 'email' queue. The email worker extends CdpCyclotronWorkerHogFlow and calls loadHogFlows (getHogFlow on functionId), so it can't process hog-function jobs. [grounded: diff] [consensus 3/3: valid=3]
The new schema field has no min(1), unlike the init schema, so the public reschedule API accepts an empty queue name. The one current caller passes a typed queue kind, so the real-world impact is low. [grounded: diff] [consensus 3/3: valid=3]
The parsed percentage is never range-checked. A value like `*:10` (someone meaning 10%) makes `Math.random() < 10` always true, so all email traffic gets routed with no error. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
<!-- cubic:v=49dcd5be-e208-4139-8231-c29b276ce97d -->
<!-- cubic:review-run=9809bde9-63ab-506c-a9ce-e101ad6295fa -->
<!-- metadata:{"confidence":9} -->
P1: This global matcher also routes standalone destination Hog Functions, but the email worker only loads HogFlow jobs. Configured teams will have ordinary email destinations moved to `email` and failed instead of delivered; restrict routing to HogFlow invocations or make the email worker handle Hog Functions too.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At nodejs/src/cdp/services/hog-executor.service.ts, line 375:
<comment>This global matcher also routes standalone destination Hog Functions, but the email worker only loads HogFlow jobs. Configured teams will have ordinary email destinations moved to `email` and failed instead of delivered; restrict routing to HogFlow invocations or make the email worker handle Hog Functions too.</comment>
<file context>
@@ -339,10 +350,33 @@ export class HogExecutorService {
- result = await this.emailService.executeSendEmail(nextInvocation)
+ // Route to the email queue only if we're not already there
+ // and the team is configured for it. Otherwise send inline.
+ if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
+ result = this.routeEmailToQueue(nextInvocation)
+ } else {
</file context>
```
</details>
```suggestion
if (invocation.queue === 'hogflow' && this.emailQueueMatcher(nextInvocation.teamId)) {
```
<!-- cubic:v=43ba1827-c015-468b-b218-cc2fda5c9bfe -->
<!-- cubic:review-run=9809bde9-63ab-506c-a9ce-e101ad6295fa -->
<!-- metadata:{"confidence":10} -->
P2: `queueName` must reject empty strings here, just like `CyclotronV2JobInitSchema.queueName`; otherwise `reschedule({ queueName: '' })` persists an available job in a queue no worker can normally consume. Add the same `.min(1)` validation.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At nodejs/src/cdp/services/cyclotron-v2/types.ts, line 40:
<comment>`queueName` must reject empty strings here, just like `CyclotronV2JobInitSchema.queueName`; otherwise `reschedule({ queueName: '' })` persists an available job in a queue no worker can normally consume. Add the same `.min(1)` validation.</comment>
<file context>
@@ -37,6 +37,7 @@ export const CyclotronV2RescheduleOptionsSchema = z.object({
distinctId: z.string().nullish(),
personId: z.string().nullish(),
actionId: z.string().nullish(),
+ queueName: z.string().optional(),
})
</file context>
```
</details>
```suggestion
queueName: z.string().min(1).optional(),
```
<!-- cubic:v=71df084f-9101-4e23-b092-f072df4f10aa -->
<!-- cubic:review-run=9809bde9-63ab-506c-a9ce-e101ad6295fa -->
<!-- metadata:{"confidence":9} -->
P2: Validate the parsed percentage before storing it. An out-of-range value such as `CDP_EMAIL_QUEUE_ROUTING=*:10` makes `Math.random() < percentage` always true, silently routing 100% of email traffic instead of the configured rollout; reject invalid values during configuration parsing.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At nodejs/src/config/config.ts, line 169:
<comment>Validate the parsed percentage before storing it. An out-of-range value such as `CDP_EMAIL_QUEUE_ROUTING=*:10` makes `Math.random() < percentage` always true, silently routing 100% of email traffic instead of the configured rollout; reject invalid values during configuration parsing.</comment>
<file context>
@@ -142,6 +142,50 @@ export function buildIntegerMatcher(config: string | undefined, allowStar: boole
+
+ for (const part of parts) {
+ if (part.startsWith('*:')) {
+ percentage = parseFloat(part.slice(2))
+ } else {
+ const num = parseInt(part)
</file context>
```
</details>
```suggestion
const percentageValue = part.slice(2)
const parsedPercentage = Number(percentageValue)
if (
percentageValue.length === 0 ||
!Number.isFinite(parsedPercentage) ||
parsedPercentage < 0 ||
parsedPercentage > 1
) {
throw new Error(`Invalid email queue routing percentage: ${part}`)
}
percentage = parsedPercentage
```
We detected this is a high-risk PR and ran a [**free ultrareview**](https://docs.cubic.dev/ai-review/ultrareview?utm_source=github). An ultrareview is a deeper, multi-pass review that catches hard-to-find bugs a standard review can miss.
This PR appears to change concurrency-sensitive code such as locks, queues, or retries, where a missed race may only surface under production load, so a deeper multi-pass review is worth running.
Want an ultrareview on every high-risk PR? [Set up automated ultrareviews](https://www.cubic.dev/ai-review?tab=ultrareview).
<!-- cubic:review-summary:start -->
**3 issues found** across 19 files
<!-- cubic:review-summary:end -->
<details>
<summary>Prompt for AI agents (unresolved issues)</summary>
```text
Check if these issues are valid — if so, understand the root cause of each and fix them. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. If appropriate, use sub-agents to investigate and fix each issue separately.
<file name="nodejs/src/cdp/services/cyclotron-v2/types.ts">
<violation number="1" location="nodejs/src/cdp/services/cyclotron-v2/types.ts:40">
P2: `queueName` must reject empty strings here, just like `CyclotronV2JobInitSchema.queueName`; otherwise `reschedule({ queueName: '' })` persists an available job in a queue no worker can normally consume. Add the same `.min(1)` validation.</violation>
</file>
<file name="nodejs/src/cdp/services/hog-executor.service.ts">
<violation number="1" location="nodejs/src/cdp/services/hog-executor.service.ts:375">
P1: This global matcher also routes standalone destination Hog Functions, but the email worker only loads HogFlow jobs. Configured teams will have ordinary email destinations moved to `email` and failed instead of delivered; restrict routing to HogFlow invocations or make the email worker handle Hog Functions too.</violation>
</file>
<file name="nodejs/src/config/config.ts">
<violation number="1" location="nodejs/src/config/config.ts:169">
P2: Validate the parsed percentage before storing it. An out-of-range value such as `CDP_EMAIL_QUEUE_ROUTING=*:10` makes `Math.random() < percentage` always true, silently routing 100% of email traffic instead of the configured rollout; reject invalid values during configuration parsing.</violation>
</file>
```
</details>
<sub>Reply with feedback, questions, or to request a fix.<br /><br />[View guided diff](https://www.cubic.dev/pr/drape-io/crbench-cubic-2026-09/pull/119?utm_source=github&utm_medium=review_footer) | [Turn on auto-fix](https://www.cubic.dev/action/auto-fix/pr/drape-io/crbench-cubic-2026-09/119?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F119) | [Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/119/ultrareview_github_delivery_13aa5f49a22ce75fce65fd83d7af196bc702ab4182532f6c211c9d88ce191013?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F119)</sub>
<!-- cubic:review-post:ultrareview_github_delivery_13aa5f49a22ce75fce65fd83d7af196bc702ab4182532f6c211c9d88ce191013:4e45802fbc93f2b971593003be605adaf1452763:9809bde9-63ab-506c-a9ce-e101ad6295fa -->
We detected this is a high-risk PR and are running a [**free ultrareview**](https://docs.cubic.dev/ai-review/ultrareview?utm_source=github). An ultrareview is a deeper, multi-pass review that catches hard-to-find bugs a standard review can miss. We'll post the findings when it completes.
This PR appears to change concurrency-sensitive code such as locks, queues, or retries, where a missed race may only surface under production load, so a deeper multi-pass review is worth running.
Want an ultrareview on every high-risk PR? [Set up automated ultrareviews](https://www.cubic.dev/ai-review?tab=ultrareview).
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
The routing check runs in the shared executeWithAsyncFunctions, so it also covers hog destinations. There is an email destination template (email.template.ts) that calls sendEmail. When the team matches CDP_EMAIL_QUEUE_ROUTING, a Kafka-backed hog job gets queue 'email', and the Kafka queue re-produces it to `cdp_cyclotron_${msg.queue}`, which nothing consumes. The only new consumer is CdpCyclotronWorkerEmail, which extends the hogflow worker and only runs on postgres-v2. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
### Email routing drops hog destination jobs
**High Severity**
<!-- DESCRIPTION START -->
`executeWithAsyncFunctions` routes any matching `sendEmail` to the `email` queue, including hog destinations on Kafka (`queue` is `hog`) and CDP API previews that never persist a job. The email worker only consumes postgres-v2 hogflow jobs, so destination emails are published to `cdp_cyclotron_email` with no consumer, or fail hogflow load and are dropped.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: cf203174-cca2-4e15-9393-385ef9be6daf -->
<!-- LOCATIONS START
nodejs/src/cdp/services/hog-executor.service.ts#L374-L376
nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts#L6-L13
LOCATIONS END -->
<details>
<summary>Additional Locations (1)</summary>
- [`nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts#L6-L13`](https://github.com/drape-io/crbench-cursor-2026-09/blob/afd05698b89e4ff276fe88a3efee0b230c2985d4/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts#L6-L13)
</details>
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmQzZWQxOWU3LTlhMWEtNDhjNy04MDg3LWZkYzliOGU3ZTY2MyIsImVuY3J5cHRpb25LZXkiOiJ5cUFYaWdCbDJzcGFvZ1ZpbXUwQVRuQzl6LUY0aEJueVpjNGNhT1ZNNjBrIiwiYnJhbmNoIjoicHItZDRkNWZlMmU1MiIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmQzZWQxOWU3LTlhMWEtNDhjNy04MDg3LWZkYzliOGU3ZTY2MyIsImVuY3J5cHRpb25LZXkiOiJ5cUFYaWdCbDJzcGFvZ1ZpbXUwQVRuQzl6LUY0aEJueVpjNGNhT1ZNNjBrIiwiYnJhbmNoIjoicHItZDRkNWZlMmU1MiIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjExOSwiY29tbWl0U2hhIjoiYWZkMDU2OThiODllNGZmMjc2ZmU4OGEzZWZlZTBiMjMwYzI5ODVkNCIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit afd05698b89e4ff276fe88a3efee0b230c2985d4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
### Wait-until jobs stuck on email queue
**High Severity**
<!-- DESCRIPTION START -->
After an email is sent on the email worker, the hogflow stays on the `email` queue. Later `wait_until_condition` or conversion waits are parked with `queue_name` `email`, but the subscription matcher only selects `queue_name = 'hogflow'`, so matching events never wake those workflows.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: bf3fdbf4-ca1e-494e-96c6-1ce3b94410b1 -->
<!-- LOCATIONS START
nodejs/src/cdp/services/hog-executor.service.ts#L371-L379
nodejs/src/cdp/services/hogflows/actions/hog_function.ts#L56-L60
LOCATIONS END -->
<details>
<summary>Additional Locations (1)</summary>
- [`nodejs/src/cdp/services/hogflows/actions/hog_function.ts#L56-L60`](https://github.com/drape-io/crbench-cursor-2026-09/blob/afd05698b89e4ff276fe88a3efee0b230c2985d4/nodejs/src/cdp/services/hogflows/actions/hog_function.ts#L56-L60)
</details>
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjI2MzIyM2IyLWE2MjQtNGU2Ny05YTA2LTFjMGZmOGE0ZGRmYiIsImVuY3J5cHRpb25LZXkiOiJKTzZuVnIyQjU3UWxrY1pYQzhPempBT0FfWndlTWt2NGRIM1hNaGItdUh3IiwiYnJhbmNoIjoicHItZDRkNWZlMmU1MiIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjI2MzIyM2IyLWE2MjQtNGU2Ny05YTA2LTFjMGZmOGE0ZGRmYiIsImVuY3J5cHRpb25LZXkiOiJKTzZuVnIyQjU3UWxrY1pYQzhPempBT0FfWndlTWt2NGRIM1hNaGItdUh3IiwiYnJhbmNoIjoicHItZDRkNWZlMmU1MiIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjExOSwiY29tbWl0U2hhIjoiYWZkMDU2OThiODllNGZmMjc2ZmU4OGEzZWZlZTBiMjMwYzI5ODVkNCIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit afd05698b89e4ff276fe88a3efee0b230c2985d4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_REVIEW -->
Cursor Bugbot has reviewed your changes using high effort and found 2 potential issues.
<!-- BUGBOT_FIX_ALL -->
<a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aTpjYzRkMzNlNS0zMzYzLTRmZTMtODcxZS0zNWU0MTBjYTNiYzgiLCJlbmNyeXB0aW9uS2V5IjoiRkZhcEgyWHhOTGkzVTZKSkRoc2pvT2NZLXBJZzFINWswN3pmUzNocjFVcyIsImJyYW5jaCI6InByLWQ0ZDVmZTJlNTIiLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>
<!-- /BUGBOT_FIX_ALL -->
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN -->
<sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END -->
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit afd05698b89e4ff276fe88a3efee0b230c2985d4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
postHogflowInvocation runs executeCurrentAction directly. For a routed team, a non-mocked sendEmail is now handed to routeEmailToQueue, which returns finished:false and only changes the queue. Nothing queues that result, yet the API reports success. Before this change the email was sent inline. [grounded: diff-section nodejs/src/cdp/services/hog-executor.service.ts] [consensus 3/3: valid=3]
In the v1 queue, invocationToCyclotronJobUpdate reuses the initial job mapping, which sets queueName to invocation.queue. Routed jobs therefore get queue_name='email' in the v1 DB. The email worker only polls the postgres-v2 queue, so those jobs are stranded. [grounded: tree nodejs/src/cdp/services/hogflows/actions/hog_function.ts] [consensus 3/3: valid=3]
The new matcher does no validation. '*:10' makes Math.random() < 10 always true, so everything is routed, and '*:abc' gives NaN, so nothing is routed, with no warning either way. This only happens when the config is wrong. [grounded: diff-section nodejs/src/config/config.ts] [consensus 3/3: valid=3]
executeWithAsyncFunctions is also used by CdpCyclotronWorker for hog-function destinations, and an email destination template uses sendEmail. A 'hog' queue invocation for a routed team is set to queue 'email', and the kafka queue republishes it to the topic cdp_cyclotron_email. Nothing consumes that topic, because the email worker reads postgres-v2 and expects hogflows. The email is never sent. [grounded: diff-section nodejs/src/cdp/services/hog-executor.service.ts] [consensus 3/3: valid=3]
raw comments (5)
**Workflow test runs skip the real email send**
When a team is listed in `CDP_EMAIL_QUEUE_ROUTING`, the hogflow test-invocation API routes a non-mocked email step to the queue. The result is never queued, so no email goes out, yet the API reports success. Disable queue routing for API and test invocations, e.g. with an executor option.
```
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/services/hog-executor.service.ts:376 result = this.routeEmailToQueue(nextInvocation)
nodejs/src/cdp/cdp-api.ts:570 res.json({
nodejs/src/cdp/cdp-api.ts:571 nextActionId: result.invocation.state.currentAction?.id,
nodejs/src/cdp/cdp-api.ts:572 status: result.error ? 'error' : 'success',
nodejs/src/cdp/cdp-api.ts:573 errors: result.error ? [result.error] : [],
nodejs/src/cdp/cdp-api.ts:574 logs: [...result.logs, ...logs],
nodejs/src/cdp/cdp-api.ts:575 variables: result.invocation.state.variables ?? {},
nodejs/src/cdp/cdp-api.ts:576 execResult: result.execResult ?? null,
nodejs/src/cdp/cdp-api.ts:577 })
```
**Legacy PG hogflow emails stranded in email queue**
The legacy Postgres v1 drain worker also routes matched emails, writing `queue_name='email'` into the v1 DB. The email worker only polls postgres-v2, so these jobs are never picked up. Skip routing unless `queueSource` is postgres-v2.
```
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/services/hog-executor.service.ts:376 result = this.routeEmailToQueue(nextInvocation)
nodejs/src/cdp/services/hogflows/actions/hog_function.ts:58 result.invocation.queue = functionResult.invocation.queue
nodejs/src/cdp/services/hogflows/actions/hog_function.ts:59 result.invocation.queueParameters = functionResult.invocation.queueParameters
nodejs/src/server.ts:275 if (capabilities.cdpCyclotronWorkerEmail) {
nodejs/src/server.ts:276 serviceLoaders.push(async () => {
nodejs/src/server.ts:277 // Dedicated queue instance — see note on cdpCyclotronWorkerHogFlow above.
nodejs/src/server.ts:278 const queue = new CyclotronJobQueuePostgresV2(this.config.CONSUMER_BATCH_SIZE, this.config)
nodejs/src/server.ts:279 const worker = new CdpCyclotronWorkerEmail(this.config, cdpDeps!, queue)
nodejs/src/server.ts:280 await worker.start()
```
**Workflows never leave the email queue**
After the email worker sends, the flow keeps running with `queue='email'`, so later delays, waits and emails are rescheduled into the email queue. This contradicts the `routeEmailToQueue` comment saying the job returns to its original queue. After the send, set the hogflow result's queue back to `queueMetadata.originQueue`.
```
nodejs/src/cdp/services/hogflows/actions/hog_function.ts:58 result.invocation.queue = functionResult.invocation.queue
nodejs/src/cdp/services/hogflows/actions/hog_function.ts:59 result.invocation.queueParameters = functionResult.invocation.queueParameters
nodejs/src/cdp/services/hogflows/actions/hog_function.ts:60 result.invocation.queueMetadata = functionResult.invocation.queueMetadata
nodejs/src/cdp/services/hog-executor.service.ts:418 private routeEmailToQueue(
nodejs/src/cdp/utils/invocation-utils.ts:44 queue: params.queue ?? invocation.queue,
nodejs/src/cdp/services/hog-executor.service.ts:413 /**
nodejs/src/cdp/services/hog-executor.service.ts:414 * Routes an email send to the dedicated email queue instead of sending inline.
nodejs/src/cdp/services/hog-executor.service.ts:415 * The email worker will pick this up, send via SES, and return the job to the
nodejs/src/cdp/services/hog-executor.service.ts:416 * original queue so the workflow can continue.
nodejs/src/cdp/services/hog-executor.service.ts:417 */
```
**Percent rollout misread when given as 10**
The `*:` value is treated as a fraction, but nothing checks its range. Writing `*:10` (meaning 10%) or `*:abc` gives 10 (route everything) or NaN (route nothing), with no warning. Reject or clamp values outside 0–1 and NaN, and log a warning.
```
nodejs/src/config/config.ts:167 for (const part of parts) {
nodejs/src/config/config.ts:168 if (part.startsWith('*:')) {
nodejs/src/config/config.ts:169 percentage = parseFloat(part.slice(2))
nodejs/src/config/config.ts:170 } else {
nodejs/src/config/config.ts:182 if (percentage > 0) {
nodejs/src/config/config.ts:183 return Math.random() < percentage
nodejs/src/config/config.ts:184 }
```
**Email destination sends dropped for routed teams**
Routing applies to all hog functions, not just workflows. An email destination on the kafka `hog` queue is requeued to `email`, which only the postgres-v2 hogflow email worker reads, so it never sends. Only route when the invocation is a hogflow (`invocation.queue === 'hogflow'`).
```
nodejs/src/cdp/services/hog-executor.service.ts:372 } else if (queueParamsType === 'email') {
nodejs/src/cdp/services/hog-executor.service.ts:373 // Route to the email queue only if we're not already there
nodejs/src/cdp/services/hog-executor.service.ts:374 // and the team is configured for it. Otherwise send inline.
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/services/job-queue/job-queue-kafka.ts:125 topic: `cdp_cyclotron_${msg.queue}`,
nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts:35 const hogFlow = await this.hogFlowManager.getHogFlow(item.functionId)
nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts:36 if (!hogFlow || !team) {
nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts:37 logger.error('⚠️', 'Error finding hog flow', {
nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts:38 id: item.functionId,
nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts:39 })
```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
The check runs in the shared HogExecutorService. Plain hog functions on the 'hog' queue can send email (there is a destination email template), so for matched teams they get routed to 'email'. That queue is only consumed by CdpCyclotronWorkerEmail, which extends the hogflow worker and loads hogflows. [grounded: diff] [consensus 3/3: valid=3]
Percentages are not validated, so '*:10' makes `Math.random() < 10` always true and routes every team. A bare '*' inside a list fails parseInt and is silently dropped. This is real but only happens with a misconfiguration. [grounded: diff] [consensus 3/3: valid=3]
The hogflow executor builds its result from the dequeued invocation, so queue stays 'email'. After the send finishes, nothing restores originQueue, so later delays reschedule the job with queue_name 'email' and the rest of the workflow keeps running on the email worker. [grounded: diff] [consensus 3/3: valid=3]
The legacy-pg hogflow worker uses the same executor, and its update writes `queueName: invocation.queue` (job-queue-postgres.ts:284). That puts jobs on an 'email' queue in the legacy backend, while the email worker only consumes postgres-v2, so those jobs are stranded. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
**Plain hog function emails routed to hogflow-only queue**
The routing check also applies to the 'hog' queue consumer, CDP API test invoke and batch export. Their invocations go to the 'email' queue, whose worker looks the function up as a hogflow. Only route to the email queue when the invocation's queue is 'hogflow'.
```
nodejs/src/cdp/services/hog-executor.service.ts:372 } else if (queueParamsType === 'email') {
nodejs/src/cdp/services/hog-executor.service.ts:373 // Route to the email queue only if we're not already there
nodejs/src/cdp/services/hog-executor.service.ts:374 // and the team is configured for it. Otherwise send inline.
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts:7 export class CdpCyclotronWorkerEmail extends CdpCyclotronWorkerHogFlow {
```
**Routing matcher silently misparses some config values**
`buildIntegerMatcherWithPercentage` treats a bare `*` in a list as nothing, and accepts any `*:N` unchecked. A `*:10` (meant as 10%) is greater than 1, so every call matches and all email goes to the queue. Treat a bare '*' part as 100%, and reject or clamp percentages outside 0-1.
```
nodejs/src/config/config.ts:161 }
nodejs/src/config/config.ts:163 const parts = config.split(',').map((s) => s.trim())
nodejs/src/config/config.ts:164 const teamIds = new Set<number>()
nodejs/src/config/config.ts:165 let percentage = 0
nodejs/src/config/config.ts:178 return (teamId: number) => {
nodejs/src/config/config.ts:179 if (teamIds.has(teamId)) {
nodejs/src/config/config.ts:180 return true
nodejs/src/config/config.ts:181 }
nodejs/src/config/config.ts:182 if (percentage > 0) {
```
**Workflow stays on email queue after send**
After the email worker sends, nothing moves the job back to its origin queue. `executeSendEmail` returns finished with queue still `email`, so later delays reschedule into the email queue. When the send finishes in the email worker, set the queue back to originQueue (default 'hogflow').
```
nodejs/src/cdp/services/hog-executor.service.ts:372 } else if (queueParamsType === 'email') {
nodejs/src/cdp/services/hog-executor.service.ts:373 // Route to the email queue only if we're not already there
nodejs/src/cdp/services/hog-executor.service.ts:374 // and the team is configured for it. Otherwise send inline.
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/services/messaging/email.service.ts:74 const result = createInvocationResult<CyclotronJobInvocationHogFunction>(
nodejs/src/cdp/services/messaging/email.service.ts:75 invocation,
nodejs/src/cdp/services/messaging/email.service.ts:76 {},
nodejs/src/cdp/services/messaging/email.service.ts:77 {
nodejs/src/cdp/services/messaging/email.service.ts:78 finished: true,
nodejs/src/cdp/services/messaging/email.service.ts:79 }
```
**Email routing ignores jobs from non-v2 queues**
The routing check never looks at `invocation.queueSource`. Hogflow jobs drained from legacy postgres (or kafka) are rescheduled onto an 'email' queue in that same backend. Only the email worker, which reads postgres-v2, consumes it. Only route to 'email' when invocation.queueSource is 'postgres-v2'.
```
nodejs/src/cdp/services/hog-executor.service.ts:372 } else if (queueParamsType === 'email') {
nodejs/src/cdp/services/hog-executor.service.ts:373 // Route to the email queue only if we're not already there
nodejs/src/cdp/services/hog-executor.service.ts:374 // and the team is configured for it. Otherwise send inline.
nodejs/src/cdp/services/hog-executor.service.ts:375 if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
nodejs/src/cdp/utils/invocation-utils.ts:46 queueSource: invocation.queueSource,
nodejs/src/cdp/services/job-queue/job-queue-postgres.ts:187 public async queueInvocationResults(invocationResults: CyclotronJobInvocationResult[]) {
nodejs/src/cdp/services/job-queue/job-queue-postgres.ts:188 invocationResults = this.sanitizer.sanitizeResults(invocationResults)
nodejs/src/cdp/services/job-queue/job-queue-postgres.ts:189 const worker = this.getCyclotronWorker()
nodejs/src/cdp/services/job-queue/job-queue-postgres.ts:190 await Promise.all(
nodejs/src/cdp/services/job-queue/job-queue-postgres.ts:191 invocationResults.map((item) => {
```
**Parked workflows on email queue never woken**
The subscription matcher only looks at parked jobs with `queue_name = 'hogflow'`. The change lets workflow jobs live on the new `email` queue, so a workflow waiting there on a wait-until-condition or delay is invisible to the matcher. Include 'email' in the matcher's queue_name filter, e.g. `queue_name IN ('hogflow','email')`.
```
nodejs/src/cdp/types.ts:257 export const CYCLOTRON_INVOCATION_JOB_QUEUES = ['hog', 'hogoverflow', 'hogflow', 'email'] as const
nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts:209 queueName: result.invocation.queue,
nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts:268 AND queue_name = 'hogflow'
nodejs/src/cdp/consumers/cdp-hogflow-subscription-matcher.consumer.ts:276 AND queue_name = 'hogflow'
```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✗ | — |
HogExecutorConfig now requires emailQueueRouting, but four call sites in the repo still build the config without it: src/cdp/templates/test/test-helpers.ts and three test files (hog-executor, hogflow-executor, hog_function). nodejs/tsconfig.json includes src and tests with strict mode on, so the type check fails. [grounded: diff] [consensus 3/3: valid=3]
The routing check only looks at team and current queue, and executeWithAsyncFunctions is shared by the Kafka-backed CdpCyclotronWorker and the legacy PG hogflow worker. A job routed to 'email' on the Kafka backend gets produced to cdp_cyclotron_email. The new email worker only consumes the postgres-v2 queue, so those jobs are never picked up. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Node.js build fails**
Making `emailQueueRouting` required breaks four existing `HogExecutorService` constructors in tests and test helpers. Their config objects still omit this field. Because `tsconfig.json` includes `src` and `tests`, both the TypeScript check and build fail. Update those constructors or make the field optional with the existing disabled default.
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Routed emails get stuck**
When `CDP_EMAIL_QUEUE_ROUTING` matches a team, this branch routes every email job to `email`. But the email worker only reads Postgres-v2 jobs. Standalone functions publish their unfinished jobs to Kafka's `cdp_cyclotron_email` topic, which the worker never reads. Legacy workflow jobs likewise stay in their old Postgres backend. These emails are never sent, and the jobs cannot continue.
Restrict routing to Postgres-v2 workflows, or add the missing handoff and support for standalone functions.
<!-- greptile_summary -->
<h2><a href="https://app.greptile.com/api/retrigger?id=78417056"><picture><source media="(prefers-color-scheme: dark)" srcset="https://greptile-static-assets.s3.amazonaws.com/badges/RetriggerDark.svg?v=2"><source media="(prefers-color-scheme: light)" srcset="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"><img alt="Retrigger" src="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2" align="right"></picture></a>Confidence Score: 3/5</h2>
**[High impact]** Adds email queue routing to the job processing system\.
Fix the build and restrict email routing to supported jobs before merging.
<h2>Findings</h2>
1. <img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"> **Node\.js build fails** <a href="https://github.com/drape-io/crbench-greptile-2026-09/pull/119#discussion_r4238449631">▶</a>
2. <img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"> **Routed emails get stuck** <a href="https://github.com/drape-io/crbench-greptile-2026-09/pull/119#discussion_r4238449635">▶</a>
<h3>Summary</h3>
This PR adds a dedicated email worker and configurable email routing.
- Selected workflow emails run on a dedicated queue.
- The local Cyclotron janitor uses its own database setting and port.
<details open><summary><h3>Diagram</h3></summary>
```mermaid
%%{init: {'theme': 'neutral'}}%%
flowchart TD
H[Postgres-v2 workflow job] --> R{Email routing enabled?}
K[Kafka standalone job] --> R
L[Legacy Postgres workflow job] --> R
R -->|No| I[Send inline]
R -->|Yes, Postgres-v2| E[Postgres-v2 email worker]
R -->|Yes, Kafka| KT[Kafka email topic: no registered reader]
R -->|Yes, legacy Postgres| LT[Legacy email queue: no registered reader]
E -->|Fetch requested| H
```
</details>
<!-- greptile_confidence_score:3 -->
<sub>Reviews (1) · Last reviewed commit: ["Update bin/start-rust-service and 18 mor..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/549e4924a1c412f0eb2ec7a83ce2dadd8ebc8bab) · [Reviewed by Greptile](https://www.greptile.com/?utm_source=greptile_expert&utm_medium=github&utm_campaign=code_reviews&utm_content=drape-io/crbench-greptile-2026-09)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
The routing gate only checks that the queue is not 'email' and that the team matches. Plain hog-function workers (queue 'hog', Kafka) and CDP API test runs therefore also get routeEmailToQueue results. Either nothing persists those results, or they go to a queue that nothing consumes, so the email is not sent when it was previously sent inline. [grounded: diff] [consensus 3/3: valid=3]
hog_function.ts now copies the 'email' queue onto the hogflow invocation. The legacy CyclotronJobQueuePostgres update writes queueName: invocation.queue, and the email worker only consumes postgres-v2, so legacy-pg jobs for routed teams get stranded. [grounded: diff] [consensus 3/3: valid=3]
Same issue as finding 1: legacy v1 invocationToCyclotronJobUpdate persists queueName 'email', and no v1 consumer exists for that queue, so the workflow stalls. [grounded: diff] [consensus 3/3: valid=3]
The CDP API's postHogflowInvocation runs executeCurrentAction on a fresh hogflow invocation and never enqueues the result. For matching teams with real async functions, the email is routed away and never sent, while the response still reports success. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
[P2] Email queue routing also catches API test runs and plain hog-function destinations
The routing check only asks whether `invocation.queue !== 'email'` and whether the team matches. Every `HogExecutorService.executeWithAsyncFunctions` caller passes that check, not just the hogflow worker. For a team in `CDP_EMAIL_QUEUE_ROUTING` (or every team with `*`), there are two failure cases. (1) CDP API test runs with real async functions (`cdp-api.ts` hogflow `executeCurrentAction` around line 568, and hog-function test around line 429) get back the `routeEmailToQueue` result. The API never enqueues that result, so no email is sent, yet the response still reports `status: 'success'`. (2) The regular hog-function worker (`CdpCyclotronWorker`, `hog` queue) running a destination that calls `sendEmail` (e.g. `template-email`) moves the job to the `email` queue. On Kafka that means producing to a `cdp_cyclotron_email` topic. On postgres-v2 the job reaches `CdpCyclotronWorkerEmail`, which loads every job as a hogflow; `getHogFlow(functionId)` then fails and the job is dequeued as failed. Either way the email is dropped. Routing should only apply in the hogflow worker context, i.e. when the origin queue is `hogflow`.
[P2] Legacy-pg hogflow worker can route emails to a queue nothing consumes
`hog_function.ts` now copies the function result's `queue` onto the hogflow invocation. The legacy `CyclotronJobQueuePostgres` writes `queueName: invocation.queue` when it updates a job. So when `CdpCyclotronWorkerHogFlowLegacyPg` runs with `CDP_EMAIL_QUEUE_ROUTING` matching a team, the job moves to `email` in the legacy cyclotron DB. The new `CdpCyclotronWorkerEmail` only consumes the postgres-v2 queue (server.ts), so these workflow runs stall there indefinitely. This only happens if the routing env var is set in the deployment that runs the legacy worker, which is easy to do if config is shared. The routing should be gated on the postgres-v2 queue source (`queueSource`) or turned off for the legacy worker.
[P2] Workflows stay on the email queue after the email is sent
When the email worker picks up a routed job, `invocation.queue` is `'email'`, so the email is sent inline. The hog function then finishes, and the hogflow executor moves on to the next actions in the same worker. The invocation was cloned with queue `'email'` and nothing resets it to `queueMetadata.originQueue`. As a result, every later step runs and is rescheduled on the `email` queue: delays, wait-until, branches, and further emails. Only a later `fetch` moves the job back to `hogflow`. This contradicts the `routeEmailToQueue` docstring, which says the job returns to the original queue "so the workflow can continue". In practice the email workers end up running whole workflows, including long-lived delayed jobs, for any team matched by `CDP_EMAIL_QUEUE_ROUTING`. That defeats the purpose of a dedicated email queue. Once the email has been sent inline from the email queue, the result should be routed back to `originQueue`.
[P2] Legacy v1 Postgres hogflow drain can strand jobs on an `email` queue nobody consumes
The handler now copies `functionResult.invocation.queue` onto the hogflow invocation. The `HogExecutorService` that runs inside `CdpCyclotronWorkerHogFlowLegacyPg` uses the same `emailQueueMatcher`. When that worker processes a matching team's email action, it routes the job to `'email'`. `CyclotronJobQueuePostgres.queueInvocationResults` then writes `queueName: 'email'` into the legacy v1 cyclotron DB (`invocationToCyclotronJobUpdate`). But `CdpCyclotronWorkerEmail` only consumes from the v2 queue, so in the legacy DB the job is never picked up again. The email is not sent and the workflow silently stalls. This only affects jobs still being drained from v1 while routing is enabled for their team. Either the legacy path should send inline or the routing should be skipped for postgres-v1-sourced invocations.
[P2] Hogflow test invocations from the CDP API stop sending emails for routed teams
The routing check only looks at `invocation.queue !== 'email'` and the team matcher. It also applies to `CdpApi`'s hogflow test endpoint, which calls `hogFlowExecutor.executeCurrentAction` with a fresh `'hogflow'` invocation, and to the hog function test path that calls `executeWithAsyncFunctions`. If the API process has `CDP_EMAIL_QUEUE_ROUTING` set and `mock_async_functions` is false, a test email for a matching team goes through `routeEmailToQueue`. Nothing persists that result, so no email is sent. The response still reports `success` with only an `email_queued` metric. Before this change the email was sent inline. Routing should be limited to jobs that are actually persisted by a queue worker, for example by skipping it when the invocation has no `queueSource`.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
The routing check has no hogflow guard. A standalone hog function on the 'hog' queue for a matched team gets rerouted to 'email'. The Kafka queue then produces it to cdp_cyclotron_email, which no worker consumes (the email worker reads postgres-v2 and loads hog flows by id). The email is never sent. The finding's mechanism is slightly off, but the claimed failure happens. [grounded: diff] [consensus 3/3: valid=3]
The queue is copied back only when the function is unfinished. After the email worker sends the email (finished), the hogflow invocation keeps queue 'email', and reschedule now writes queueName=result.invocation.queue. originQueue is read only on the fetch path, so the workflow stays on the email queue. [grounded: tree nodejs/src/cdp/services/hogflows/actions/hog_function.ts] [consensus 3/3: valid=3]
executeWithAsyncFunctions is shared by CdpCyclotronWorker, CdpApi test invocations and batch export, and routing applies to all of them for matched teams. Standalone emails are rerouted to a queue that nothing processes for hog functions, and API callers get unfinished results with no send. [grounded: diff] [consensus 3/3: valid=3]
raw comments (4)
[P2] Workflow stays pinned to the email queue after the email is sent
When an email is routed to the 'email' queue, the hogflow invocation's `queue` is set to 'email' (and `queueMetadata.originQueue` is stored). Once the email worker sends the email, `executeSendEmail` returns `finished: true`, so the handler takes the `nextAction` path. That path never restores `queue` to `originQueue`, and `cloneInvocation` keeps the 'email' queue and its stale metadata. Every later delay or action reschedule is therefore written back with `queueName: 'email'` (job-queue-postgres-v2.ts:209), and the rest of the workflow runs on the email workers. This contradicts the `routeEmailToQueue` docstring ("return the job to the original queue"), and `originQueue` is only read for the fetch case. It affects any team with `CDP_EMAIL_QUEUE_ROUTING` enabled. The dedicated queue ends up carrying all post-email workflow traffic, including long delays, instead of just the send. If the email worker is down or lagging, the whole workflow stalls, not only the send.
[P2] Email routing is applied to non-workflow hog functions, whose 'email' jobs the workflow-only email worker can't load
The routing check only looks at the invocation's queue and the team matcher. It does not check that the invocation comes from a hogflow. A standalone hog function in the 'hog' or 'hogoverflow' queue that sends an email, for a team matched by `CDP_EMAIL_QUEUE_ROUTING`, is rewritten to `queue: 'email'`. `CdpCyclotronWorkerEmail` extends the hogflow consumer and calls `loadHogFlows`, which looks up `item.functionId` as a hog flow. A hog function id yields no hog flow, so the job is dequeued as failed. The email is never sent, and the VM state is lost. This only happens when routing is enabled for a team that sends email from non-workflow hog functions.
[P2] Workflow is never returned from the email queue after the send
routeEmailToQueue` only records `originQueue` and says the job returns to the original queue after sending. Nothing does that. After `executeSendEmail` finishes, `hog_function.ts` only copies queue/metadata for unfinished results, so the hogflow invocation keeps `queue: 'email'`. The workflow then continues on the email worker, and every delay or wait reschedules with `queueName: 'email'` via the new `reschedule` update. `originQueue` is read only when a fetch happens on the email queue. For a team with `CDP_EMAIL_QUEUE_ROUTING` enabled, each workflow that sends one email stays on the email queue for the rest of its life. Later non-fetch actions run on the email pods, and later emails bypass routing because `invocation.queue === 'email'`. The email worker therefore ends up with whole workflows, not just sends. The fix is to reset the queue to `originQueue` once the email has been sent
[P2] Standalone hog function emails can be routed to the email queue and dropped
The email routing applies to every caller of `executeWithAsyncFunctions`, not just hogflows. `sendEmail` is registered globally in `async-functions/send-email.ts`, so a standalone destination run by `CdpCyclotronWorker` on the `hog` queue also hits it. With `CDP_EMAIL_QUEUE_ROUTING` enabled for the team, that job is rescheduled onto the `email` queue. `CdpCyclotronWorkerEmail` extends the hogflow consumer, and `loadHogFlows` looks up `functionId` as a hog flow. A hog function id has no hog flow, so the job is dequeued and the email is never sent. The same routing also returns an unfinished result with no send to the batch-export service and the CDP API test-invocation path. Routing should be limited to hogflow invocations, or the email worker should handle plain hog function invocations
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
CdpApi postFunctionInvocation calls hogExecutor.executeWithAsyncFunctions without mocks and never enqueues the result. For a team that matches the routing config, the email is routed to the queue, the loop exits, no error is recorded, and the API reports 'success' even though no email is sent. [grounded: diff] [consensus 3/3: valid=3]
The routing has no check that the invocation is a HogFlow. A standalone hog destination email (the email template uses sendEmail) running on the kafka 'hog' worker gets requeued to topic cdp_cyclotron_email, which nothing consumes. The email worker uses only postgres-v2 and only loads HogFlows, so the email is dropped. [grounded: diff] [consensus 3/3: valid=3]
The legacy v1 postgres worker uses the same executor, and hog_function.ts now copies the queue onto the flow invocation. The v1 update copies invocation.queue into queueName, so the job moves to the v1 'email' queue. The new email worker only consumes postgres-v2, so the job is stranded. [grounded: tree nodejs/src/cdp/services/job-queue/job-queue-postgres.ts] [consensus 3/3: valid=3]
raw comments (6)
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required">
1\. Real email tests report success without sending <code>🐞 Bug</code> <code>≡ Correctness</code>
<pre>
<b><i>routeEmailToQueue</i></b> returns an unfinished invocation rather than sending the email, while the direct
invocation API endpoints return executor results without enqueuing them. When a matching team runs a
non-mocked function or workflow test, the API can return a success response even though no worker
receives the email job.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
Direct, non-mocked invocation endpoints do not enqueue unfinished results, so routing their emails silently prevents delivery.
## Fix Focus Areas
- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
- nodejs/src/cdp/cdp-api.ts[405-452]
- nodejs/src/cdp/cdp-api.ts[563-578]
## Recommended Fix
Pass an explicit execution context to the executor and bypass email-queue routing for direct API invocations, so non-mocked tests retain inline delivery. Cover both function and workflow invocation endpoints.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
[Dismiss ↗](https://app.qodo.ai/findings/4972fadc-3b59-4e3c-b647-cc468e1e8147?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/4972fadc-3b59-4e3c-b647-cc468e1e8147)
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required">
2\. Email destinations and test sends are dropped for routed teams <code>🐞 Bug</code> <code>≡ Correctness</code>
<pre>
<b><i>executeWithAsyncFunctions</i></b> routes every matching <b><i>email</i></b> async call to the dedicated queue, even
when the invocation is not a HogFlow. Because the shared executor also serves standalone Hog
Functions, CdpApi test invocations and batch exports, those calls can reach a Kafka topic with no
consumer or an email worker that loads only HogFlows; a standalone function fails before sending,
while an API test can report success with a non-finished result and no email.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
Email queue routing in the shared HogExecutorService applies to callers beyond HogFlows. Non-HogFlow emails can be requeued where they are not consumed or sent to a HogFlow-only worker, leaving them unsent; the CdpApi test path can report success without completing the send.
## Fix Focus Areas
- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
- nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts[7-13]
## Recommended Fix
Route to the email queue only when `invocation.queue === 'hogflow'`, or require an explicit option supplied by the HogFlow worker, such as `options.allowEmailQueueRouting`. Keep sends inline for every other caller, including standalone Hog Functions and CdpApi tests. Alternatively, make the email worker load and execute both standalone Hog Function and HogFlow invocations while preserving each job type's continuation path.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
[Dismiss ↗](https://app.qodo.ai/findings/5aab5c8f-e8ce-4dea-ad7c-fed012b9ffc3?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/5aab5c8f-e8ce-4dea-ad7c-fed012b9ffc3)
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
3\. Legacy postgres workflow jobs get stranded on an unconsumed email queue <code>🐞 Bug</code> <code>☼ Reliability</code>
<pre>
The <b><i>CdpCyclotronWorkerHogFlowLegacyPg</i></b> drain worker uses the same <b><i>HogExecutorService</i></b>, so routed
teams get their emails rewritten to queue <b><i>email</i></b>. <b><i>CyclotronJobQueuePostgres</i></b> keeps `queueName:
invocation.queue` when it writes results back to the v1 cyclotron DB. The new email worker only
consumes from the Postgres-v2 queue, so these workflow jobs sit in v1 <b><i>email</i></b> forever and the email
is never sent.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
The legacy v1 Postgres hogflow worker can route emails to an `email` queue that no worker consumes.
## Fix Focus Areas
- nodejs/src/cdp/services/hog-executor.service.ts[375-379]
## Recommended Fix
Route to the email queue only when `invocation.queueSource === 'postgres-v2'`, or when the caller explicitly enables it. Otherwise send inline.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
[Dismiss ↗](https://app.qodo.ai/findings/ceca9421-fe9d-471b-9dd0-21d5d81ae9c4?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/ceca9421-fe9d-471b-9dd0-21d5d81ae9c4)
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
4\. Workflows stay on the email queue after the email is sent <code>🐞 Bug</code> <code>≡ Correctness</code>
<pre>
Once the email worker sends the email inline, nothing sets the hogflow invocation's <b><i>queue</i></b> back to
the origin queue. <b><i>createInvocationResult</i></b> keeps <b><i>queue: 'email'</i></b>, and <b><i>HogFunctionHandler</i></b> copies
that queue onto the flow. The rest of the workflow (later actions, delays and wait-until
reschedules) then runs and is persisted on the <b><i>email</i></b> queue, even though the <b><i>routeEmailToQueue</i></b>
docstring says the job returns to the original queue. Only a later fetch sends it back, so the
dedicated email workers end up carrying general workflow load.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
After the email worker sends an email, the hogflow invocation keeps `queue: 'email'`, so the rest of the workflow runs on the email queue.
## Fix Focus Areas
- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
- nodejs/src/cdp/services/hogflows/actions/hog_function.ts[54-63]
## Recommended Fix
When `invocation.queue === 'email'` and the email has been sent, set the result's queue to `queueMetadata.originQueue ?? 'hogflow'`. Then break out and reschedule so that later actions run on the hogflow worker. Alternatively, have the hogflow handler or executor reset the queue to the origin once the function finishes.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
[Dismiss ↗](https://app.qodo.ai/findings/77990fa2-b192-4900-a435-48cd6a42748b?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/77990fa2-b192-4900-a435-48cd6a42748b)
<h3>PR Summary by Qodo</h3>
Route workflow emails to a dedicated Cyclotron worker
<code>✨ Enhancement</code> <code>⚙️ Configuration changes</code> <code>🕐 40+ Minutes</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<details>
<summary>AI Description</summary>
<dl>
<dd>
<br/>
><pre>
>• Add a dedicated email worker to isolate email delivery from HogFlow processing.
>• Route email jobs by team or rollout percentage, while retaining inline delivery by default.
>• Persist queue changes across workflow reschedules and track emails routed to the new queue.
></pre>
</dd>
</dl>
</details>
<details>
<summary>Diagram</summary>
<dl>
<dd>
<br/>
```mermaid
graph TD
Flow["HogFlow action"] --> Exec["Hog executor"] --> Match{"Email rollout?"}
Match -->|Yes| EmailQ["Email queue"] --> EmailW["Email worker"] --> Send["Email service"]
Match -->|No| Send
EmailW -->|Fetch handoff| HogQ["Hogflow queue"] --> HogW["HogFlow worker"] --> Exec
Send -->|Routed continuation| HogQ
```
</dd>
</dl>
</details>
<details>
<summary>High-Level Assessment</summary>
<dl>
<dd>
<br/>
>The following are alternative approaches to this PR:
<details>
<summary><b>1. Separate email jobs with completion events</b></summary>
<dl>
<dd>
- ➕ Could isolate email delivery without moving the workflow job between queues.
- ➖ Requires a new completion and correlation mechanism to resume workflow state, increasing operational complexity.
</dd>
</dl>
</details>
>**Recommendation:** Reusing the existing HogFlow worker and rescheduling the same Cyclotron job is the simpler fit for this rollout. Before broad enablement, verify queue handoffs, email failures, and workflow continuation with tests; those paths are not covered by changes in this PR.
</dd>
</dl>
</details>
<details>
<summary> Files changed (19) <code> +215 / -16 </code> </summary>
<dl>
<dd>
<br/>
<details>
<summary>Enhancement (14) <code> +193 / -10 </code></summary>
<dl>
<dd>
<details>
<summary>capabilities.ts<code>Enable the email worker capability</code> <code>+5/-0</code></summary>
<br/>
>Enable the email worker capability
>
><pre>
>• Adds the email worker to the CDP workflows capability set and maps its standalone server mode to that capability.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-c30a1ecfc0137a1b006c1b44b4650feba5af94275988118b51f80c744cee88a0'>nodejs/src/capabilities.ts</a>
</details>
<details>
<summary>cdp-services.ts<code>Expose email service and pass routing configuration</code> <code>+4/-0</code></summary>
<br/>
>Expose email service and pass routing configuration
>
><pre>
>• Makes EmailService available through core CDP services and passes the email queue routing setting into the Hog executor.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-77814020708987f653491b9698ece82ea01b3448fc0224c5edebd070d3f86e52'>nodejs/src/cdp/cdp-services.ts</a>
</details>
<details>
<summary>cdp-base.consumer.ts<code>Expose EmailService to CDP consumers</code> <code>+3/-0</code></summary>
<br/>
>Expose EmailService to CDP consumers
>
><pre>
>• Stores the shared EmailService on the consumer base for use by consumer implementations.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-f0260c5cb214e0d14e4c995c57eceac9a9213b58d0c048b208a91ea754d8de30'>nodejs/src/cdp/consumers/cdp-base.consumer.ts</a>
</details>
<details>
<summary>cdp-cyclotron-worker-email.consumer.ts<code>Add a worker for the email queue</code> <code>+14/-0</code></summary>
<br/>
>Add a worker for the email queue
>
><pre>
>• Defines an email worker that reuses HogFlow worker behavior while consuming the email queue.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-061cabcd6087f9b9ab7a34d456eca162b9f2f431dd83617ea7ac95975992cb08'>nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts</a>
</details>
<details>
<summary>hog-transformer.service.ts<code>Pass email routing into transformation execution</code> <code>+1/-0</code></summary>
<br/>
>Pass email routing into transformation execution
>
><pre>
>• Supplies the email queue rollout setting to the Hog executor created for transformations.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-f4283a97af9cf90dcc7582d7e7d0b21f5bce20330673feaa4cf76d6b57adba54'>nodejs/src/cdp/hog-transformations/hog-transformer.service.ts</a>
</details>
<details>
<summary>types.ts<code>Allow queue changes when rescheduling jobs</code> <code>+1/-0</code></summary>
<br/>
>Allow queue changes when rescheduling jobs
>
><pre>
>• Adds an optional queue name to Cyclotron V2 reschedule options.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-6d94a0022da517fe7c3eb59fa85bccfb93102c52ca2b3bba60b3c282294f1721'>nodejs/src/cdp/services/cyclotron-v2/types.ts</a>
</details>
<details>
<summary>worker.ts<code>Persist the rescheduled job queue</code> <code>+4/-0</code></summary>
<br/>
>Persist the rescheduled job queue
>
><pre>
>• Updates queue_name when a reschedule supplies a new queue, allowing an existing job to move between workers.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-d1db4cdf6536031e105049a3abae60c989a31f679825414199a8fea806ff6317'>nodejs/src/cdp/services/cyclotron-v2/worker.ts</a>
</details>
<details>
<summary>hog-executor.service.ts<code>Route email actions and track queue handoffs</code> <code>+90/-7</code></summary>
<br/>
>Route email actions and track queue handoffs
>
><pre>
>• Routes eligible email actions to the email queue, sends other emails inline, and sends fetch work encountered there back to the originating queue. Adds an email-queued counter and metric, stops execution on queue changes, and preserves the current queue on fetch retries.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-87e25ba09f2475c2d76b7c4ba5fdc463484c3d460e4b1ee9b02b57e197fd8b72'>nodejs/src/cdp/services/hog-executor.service.ts</a>
</details>
<details>
<summary>hog_function.ts<code>Preserve queue routing in workflow actions</code> <code>+3/-1</code></summary>
<br/>
>Preserve queue routing in workflow actions
>
><pre>
>• Copies the Hog function's queue and metadata into the parent workflow invocation when the action pauses, so rescheduling retains its routing decision.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-d048503e0fce6ebe985025024d8fae09b0c2accb7d77fe5d0ab6bb14f1256ab2'>nodejs/src/cdp/services/hogflows/actions/hog_function.ts</a>
</details>
<details>
<summary>job-queue-postgres-v2.ts<code>Forward invocation queue on reschedule</code> <code>+1/-0</code></summary>
<br/>
>Forward invocation queue on reschedule
>
><pre>
>• Passes the result invocation's queue name into Cyclotron V2 job rescheduling.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-1d4fa822ccead0c68db9bee9551cdf6c6f7bc4d30e28e9c389f33061d49f192a'>nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts</a>
</details>
<details>
<summary>types.ts<code>Define the email queue and queued metric</code> <code>+2/-1</code></summary>
<br/>
>Define the email queue and queued metric
>
><pre>
>• Adds email to the invocation queue names and email_queued to the app metric names.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-adb9e5ad4fa526cbe62a1ed8fd5aaac3ce8ffa3434283198fc5869126f420c9c'>nodejs/src/cdp/types.ts</a>
</details>
<details>
<summary>config.ts<code>Parse team and percentage email rollouts</code> <code>+44/-0</code></summary>
<br/>
>Parse team and percentage email rollouts
>
><pre>
>• Adds a matcher for explicit team IDs, all-team routing, and random per-call percentage sampling.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-3f5a1ff11cfbd697384635b085be71a7f1aa6da4fef7bc0e13f718011829c3a0'>nodejs/src/config/config.ts</a>
</details>
<details>
<summary>server.ts<code>Start email and HogFlow workers with separate queues</code> <code>+20/-1</code></summary>
<br/>
>Start email and HogFlow workers with separate queues
>
><pre>
>• Starts the email worker when enabled and gives both it and the HogFlow worker dedicated Postgres V2 queue instances to avoid shared consumer state.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-cf5ac11ea38c04baff9bf084e1dc3c956974cbfaf90530c6a5df49922a91bc48'>nodejs/src/server.ts</a>
</details>
<details>
<summary>types.ts<code>Declare the email worker capability</code> <code>+1/-0</code></summary>
<br/>
>Declare the email worker capability
>
><pre>
>• Adds the email worker flag to the plugin-server capabilities interface.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-eaef2a7062e7c0f9e2a8b4090c404dc476530839abb0642b2513a289e5cbf1a1'>nodejs/src/types.ts</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Bug fix (1) <code> +3 / -3 </code></summary>
<dl>
<dd>
<details>
<summary>email.service.ts<code>Support email sends without inline VM state</code> <code>+3/-3</code></summary>
<br/>
>Support email sends without inline VM state
>
><pre>
>• Pushes a response onto the VM stack only when one exists and tolerates missing Hog function metadata, supporting sends from the dedicated queue.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-7b67a28d5242ad50bd656ce0b028f8f7ea18df2129d880df7e12cd9b7fc31be6'>nodejs/src/cdp/services/messaging/email.service.ts</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Refactor (1) <code> +10 / -2 </code></summary>
<dl>
<dd>
<details>
<summary>cdp-events.consumer.ts<code>Extract queue producer lifecycle hooks</code> <code>+10/-2</code></summary>
<br/>
>Extract queue producer lifecycle hooks
>
><pre>
>• Moves producer startup and shutdown into protected methods while preserving the existing lifecycle behavior.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-4b5618539675c1833f67bae55bf180f38a718f89942bcf1d81f1250faff0b12e'>nodejs/src/cdp/consumers/cdp-events.consumer.ts</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Other (3) <code> +9 / -1 </code></summary>
<dl>
<dd>
<details>
<summary>start-rust-service<code>Separate local Cyclotron database and port settings</code> <code>+2/-1</code></summary>
<br/>
>Separate local Cyclotron database and port settings
>
><pre>
>• Uses CYCLOTRON_DATABASE_URL for the Cyclotron service and binds it to port 3310 to avoid a local port collision.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-3f5638a0ba1c9a73672d549c87985d049f54261400f47dd021a5ad8f9bcbbf8e'>bin/start-rust-service</a>
</details>
<details>
<summary>config.ts<code>Configure optional email queue rollout</code> <code>+6/-0</code></summary>
<br/>
>Configure optional email queue rollout
>
><pre>
>• Introduces CDP_EMAIL_QUEUE_ROUTING with an empty, disabled default and documents team-ID and percentage rollout formats.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-a902e641c8202aa1684973f500c499238ba21de8d4aa78238474ea68c4574612'>nodejs/src/cdp/config.ts</a>
</details>
<details>
<summary>config.ts<code>Add standalone email worker mode</code> <code>+1/-0</code></summary>
<br/>
>Add standalone email worker mode
>
><pre>
>• Defines a plugin-server mode for running the Cyclotron email worker.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-4bc7cf31623d3a986de0a37a13a6cd9e2f3420c3bacbb3839157a84e3978af5e'>nodejs/src/common/config.ts</a>
</details>
</dd>
</dl>
</details>
</dd>
</dl>
</details>
<h3>Code Review by Qodo</h3>
<code>🐞 Bugs (4)</code> <code>📘 Rule violations (0)</code> <code>📜 Skill insights (0)</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<br/>
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required">
<details>
<summary> 1. Real email tests report success without sending <code>🐞 Bug</code> <code>≡ Correctness</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
><b><i>routeEmailToQueue</i></b> returns an unfinished invocation rather than sending the email, while the direct
>invocation API endpoints return executor results without enqueuing them. When a matching team runs a
>non-mocked function or workflow test, the API can return a success response even though no worker
>receives the email job.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[nodejs/src/cdp/services/hog-executor.service.ts[R375-376]](https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-87e25ba09f2475c2d76b7c4ba5fdc463484c3d460e4b1ee9b02b57e197fd8b72R375-R376)</code>
>
>```diff
>+ if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
>+ result = this.routeEmailToQueue(nextInvocation)
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
>The routing branch only creates an unfinished result. Both direct API paths serialize their executor
>result without submitting a queue job, and the notification test form defaults to non-mocked
>execution.
></pre>
>
> <code>[nodejs/src/cdp/services/hog-executor.service.ts\[372-379\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/hog-executor.service.ts/#L372-L379)</code>
> <code>[nodejs/src/cdp/services/hog-executor.service.ts\[418-445\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/hog-executor.service.ts/#L418-L445)</code>
> <code>[nodejs/src/cdp/cdp-api.ts\[405-452\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/cdp-api.ts/#L405-L452)</code>
> <code>[nodejs/src/cdp/cdp-api.ts\[563-578\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/cdp-api.ts/#L563-L578)</code>
> <code>[products/workflows/frontend/Workflows/hogflows/panel/testing/hogFlowEditorNotificationTestLogic.ts\[163-170\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/products/workflows/frontend/Workflows/hogflows/panel/testing/hogFlowEditorNotificationTestLogic.ts/#L163-L170)</code>
> <code>[products/workflows/frontend/Workflows/hogflows/panel/testing/hogFlowEditorNotificationTestLogic.ts\[196-208\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/products/workflows/frontend/Workflows/hogflows/panel/testing/hogFlowEditorNotificationTestLogic.ts/#L196-L208)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>Direct, non-mocked invocation endpoints do not enqueue unfinished results, so routing their emails silently prevents delivery.
>
>## Fix Focus Areas
>- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
>- nodejs/src/cdp/cdp-api.ts[405-452]
>- nodejs/src/cdp/cdp-api.ts[563-578]
>
>## Recommended Fix
>Pass an explicit execution context to the executor and bypass email-queue routing for direct API invocations, so non-mocked tests retain inline delivery. Cover both function and workflow invocation endpoints.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
[Dismiss ↗](https://app.qodo.ai/findings/4972fadc-3b59-4e3c-b647-cc468e1e8147?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/4972fadc-3b59-4e3c-b647-cc468e1e8147)
<hr/>
</details>
<details>
<summary> 2. Email destinations and test sends are dropped for routed teams <code>🐞 Bug</code> <code>≡ Correctness</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
><b><i>executeWithAsyncFunctions</i></b> routes every matching <b><i>email</i></b> async call to the dedicated queue, even
>when the invocation is not a HogFlow. Because the shared executor also serves standalone Hog
>Functions, CdpApi test invocations and batch exports, those calls can reach a Kafka topic with no
>consumer or an email worker that loads only HogFlows; a standalone function fails before sending,
>while an API test can report success with a non-finished result and no email.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[nodejs/src/cdp/services/hog-executor.service.ts[R375-379]](https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-87e25ba09f2475c2d76b7c4ba5fdc463484c3d460e4b1ee9b02b57e197fd8b72R375-R379)</code>
>
>```diff
>+ if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
>+ result = this.routeEmailToQueue(nextInvocation)
>+ } else {
>+ result = await this.emailService.executeSendEmail(nextInvocation)
>+ }
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
><b><i>sendEmail</i></b> is a generic async function used by the email destination template, so standalone Hog
>Functions can produce email queue parameters, and <b><i>executeWithAsyncFunctions</i></b> is also called by the
>hog worker, CdpApi and batch exports. Standalone invocations use the <b><i>hog</i></b> queue, but the new branch
>routes matching emails to <b><i>email</i></b>; Kafka publishes to <b><i>cdp_cyclotron_${queue}</i></b>, while the email
>worker runs <b><i>loadHogFlows</i></b> and cannot load a standalone function ID as a workflow.
></pre>
>
> <code>[nodejs/src/cdp/async-functions/send-email.ts\[7-13\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/async-functions/send-email.ts/#L7-L13)</code>
> <code>[nodejs/src/cdp/templates/_destinations/email/email.template.ts\[3-18\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/templates/_destinations/email/email.template.ts/#L3-L18)</code>
> <code>[nodejs/src/cdp/cdp-api.ts\[424-430\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/cdp-api.ts/#L424-L430)</code>
> <code>[nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts\[40-56\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/consumers/cdp-cyclotron-worker.consumer.ts/#L40-L56)</code>
> <code>[nodejs/src/cdp/services/job-queue/job-queue-kafka.ts\[121-127\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/job-queue/job-queue-kafka.ts/#L121-L127)</code>
> <code>[nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts\[18-45\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts/#L18-L45)</code>
> <code>[nodejs/src/cdp/utils/invocation-utils.ts\[10-28\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/utils/invocation-utils.ts/#L10-L28)</code>
> <code>[nodejs/src/cdp/async-functions/send-email.ts\[7-14\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/async-functions/send-email.ts/#L7-L14)</code>
> <code>[nodejs/src/cdp/services/hog-executor.service.ts\[372-379\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/hog-executor.service.ts/#L372-L379)</code>
> <code>[nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts\[7-13\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts/#L7-L13)</code>
> <code>[nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts\[18-40\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts/#L18-L40)</code>
> <code>[nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts\[89-93\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts/#L89-L93)</code>
> <code>[nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts\[216-226\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts/#L216-L226)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>Email queue routing in the shared HogExecutorService applies to callers beyond HogFlows. Non-HogFlow emails can be requeued where they are not consumed or sent to a HogFlow-only worker, leaving them unsent; the CdpApi test path can report success without completing the send.
>
>## Fix Focus Areas
>- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
>- nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts[7-13]
>
>## Recommended Fix
>Route to the email queue only when `invocation.queue === 'hogflow'`, or require an explicit option supplied by the HogFlow worker, such as `options.allowEmailQueueRouting`. Keep sends inline for every other caller, including standalone Hog Functions and CdpApi tests. Alternatively, make the email worker load and execute both standalone Hog Function and HogFlow invocations while preserving each job type's continuation path.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
[Dismiss ↗](https://app.qodo.ai/findings/5aab5c8f-e8ce-4dea-ad7c-fed012b9ffc3?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/5aab5c8f-e8ce-4dea-ad7c-fed012b9ffc3)
<hr/>
</details>
<br/>
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
<details>
<summary> 3. Legacy Postgres workflow jobs get stranded on an unconsumed email queue <code>🐞 Bug</code> <code>☼ Reliability</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
>The <b><i>CdpCyclotronWorkerHogFlowLegacyPg</i></b> drain worker uses the same <b><i>HogExecutorService</i></b>, so routed
>teams get their emails rewritten to queue <b><i>email</i></b>. <b><i>CyclotronJobQueuePostgres</i></b> keeps `queueName:
>invocation.queue` when it writes results back to the v1 cyclotron DB. The new email worker only
>consumes from the Postgres-v2 queue, so these workflow jobs sit in v1 <b><i>email</i></b> forever and the email
>is never sent.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[nodejs/src/cdp/services/hog-executor.service.ts[R375-377]](https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-87e25ba09f2475c2d76b7c4ba5fdc463484c3d460e4b1ee9b02b57e197fd8b72R375-R377)</code>
>
>```diff
>+ if (invocation.queue !== 'email' && this.emailQueueMatcher(nextInvocation.teamId)) {
>+ result = this.routeEmailToQueue(nextInvocation)
>+ } else {
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
>The legacy worker is built with <b><i>CyclotronJobQueuePostgres</i></b>, and that queue maps <b><i>invocation.queue</i></b>
>to <b><i>queueName</i></b> when it writes jobs. The email worker is only ever built with
><b><i>CyclotronJobQueuePostgresV2</i></b>.
></pre>
>
> <code>[nodejs/src/server.ts\[263-272\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/server.ts/#L263-L272)</code>
> <code>[nodejs/src/server.ts\[275-283\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/server.ts/#L275-L283)</code>
> <code>[nodejs/src/cdp/services/job-queue/job-queue-postgres.ts\[280-292\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/job-queue/job-queue-postgres.ts/#L280-L292)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>The legacy v1 Postgres hogflow worker can route emails to an `email` queue that no worker consumes.
>
>## Fix Focus Areas
>- nodejs/src/cdp/services/hog-executor.service.ts[375-379]
>
>## Recommended Fix
>Route to the email queue only when `invocation.queueSource === 'postgres-v2'`, or when the caller explicitly enables it. Otherwise send inline.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
[Dismiss ↗](https://app.qodo.ai/findings/ceca9421-fe9d-471b-9dd0-21d5d81ae9c4?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/ceca9421-fe9d-471b-9dd0-21d5d81ae9c4)
<hr/>
</details>
<details>
<summary> 4. Workflows stay on the email queue after the email is sent <code>🐞 Bug</code> <code>≡ Correctness</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
>Once the email worker sends the email inline, nothing sets the hogflow invocation's <b><i>queue</i></b> back to
>the origin queue. <b><i>createInvocationResult</i></b> keeps <b><i>queue: 'email'</i></b>, and <b><i>HogFunctionHandler</i></b> copies
>that queue onto the flow. The rest of the workflow (later actions, delays and wait-until
>reschedules) then runs and is persisted on the <b><i>email</i></b> queue, even though the <b><i>routeEmailToQueue</i></b>
>docstring says the job returns to the original queue. Only a later fetch sends it back, so the
>dedicated email workers end up carrying general workflow load.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[nodejs/src/cdp/services/hogflows/actions/hog_function.ts[R57-60]](https://github.com/drape-io/crbench-qodo-2026-09/pull/119/files#diff-d048503e0fce6ebe985025024d8fae09b0c2accb7d77fe5d0ab6bb14f1256ab2R57-R60)</code>
>
>```diff
>+ // Preserve queue routing and parameters from the function result
>+ result.invocation.queue = functionResult.invocation.queue
> result.invocation.queueParameters = functionResult.invocation.queueParameters
>+ result.invocation.queueMetadata = functionResult.invocation.queueMetadata
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
><b><i>executeSendEmail</i></b> builds its result with <b><i>createInvocationResult(invocation, {})</i></b>, which keeps
><b><i>invocation.queue</i></b>. <b><i>executeCurrentAction</i></b> also clones with the same queue. The job is then
>rescheduled with <b><i>queueName: result.invocation.queue</i></b>, so after an email the job keeps being
>rescheduled on <b><i>email</i></b>.
></pre>
>
> <code>[nodejs/src/cdp/services/messaging/email.service.ts\[74-80\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/messaging/email.service.ts/#L74-L80)</code>
> <code>[nodejs/src/cdp/utils/invocation-utils.ts\[42-55\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/utils/invocation-utils.ts/#L42-L55)</code>
> <code>[nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts\[338-339\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/hogflows/hogflow-executor.service.ts/#L338-L339)</code>
> <code>[nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts\[201-209\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts/#L201-L209)</code>
> <code>[nodejs/src/cdp/services/hog-executor.service.ts\[413-417\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab/nodejs/src/cdp/services/hog-executor.service.ts/#L413-L417)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>After the email worker sends an email, the hogflow invocation keeps `queue: 'email'`, so the rest of the workflow runs on the email queue.
>
>## Fix Focus Areas
>- nodejs/src/cdp/services/hog-executor.service.ts[372-379]
>- nodejs/src/cdp/services/hogflows/actions/hog_function.ts[54-63]
>
>## Recommended Fix
>When `invocation.queue === 'email'` and the email has been sent, set the result's queue to `queueMetadata.originQueue ?? 'hogflow'`. Then break out and reschedule so that later actions run on the hogflow worker. Alternatively, have the hogflow handler or executor reset the queue to the origin once the function finishes.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
[Dismiss ↗](https://app.qodo.ai/findings/77990fa2-b192-4900-a435-48cd6a42748b?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/77990fa2-b192-4900-a435-48cd6a42748b)
<hr/>
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-context:start -->
<details><summary><strong>Context sources</strong></summary>
<div>Review mode: <code>Auto: 🧠 Deep</code>: Cross-cutting queue-routing logic spans many independent, defect-prone execution paths.</div>
<!-- qodo-context:end -->
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-daily-tip:start -->
<details>
<summary><strong>Tip of the day</strong></summary>
<br/>
<pre>💡 Did you know, you can copy the agent prompt from any finding and feed it to your IDE agent</pre>
<a href="https://docs.qodo.ai/tips-and-tricks">More tips ↗</a> | <a href="https://app.qodo.ai/configurations?tab=display-preferences">Customize Qodo ↗</a> | <a href="https://docs.qodo.ai">Qodo docs ↗</a>
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-daily-tip:end -->
<!-- https://github.com/drape-io/crbench-qodo-2026-09/commit/7d4aa5b951ee809ff0da9e98c5c30bdeb5c507ab -->
<a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>