Span Buffer Multiprocess Enhancement with Health Monitoring
Tier 2 hard python · local · real merged PR from getsentry/sentry #93824
Ground-truth defects
Because flusher processes are created via multiprocessing.get_context('spawn').Process, they are instances of multiprocessing.context.SpawnProcess, which on POSIX is not a subclass of multiprocessing.Process, so this isinstance check will always be false and hung processes won't be killed here.
Breaking out of the loop when the deadline has elapsed can skip terminating remaining flusher processes, potentially leaving them running after shutdown; consider ensuring termination is attempted even if the deadline is exceeded.
The change under review
diff
@@ -445,6 +445,17 @@ for org in organizations: # RIGHT: Use prefetch_related organizations.prefetch_related('projects') + +# WRONG: Use hasattr() for unions +x: str | None = "hello" +if hasattr(x, "replace"): + x = x.replace("e", "a") + +# RIGHT: Use isinstance() +x: str | None = "hello" +if isinstance(x, str): + x = x.replace("e", "a") + ``` ### Frontend @@ -427,7 +427,15 @@ KAFKA_CONSUMERS: Mapping[str, ConsumerDefinition] = { "topic": Topic.INGEST_SPANS, "dlq_topic": Topic.INGEST_SPANS_DLQ, "strategy_factory": "sentry.spans.consumers.process.factory.ProcessSpansStrategyFactory", - "click_options": multiprocessing_options(default_max_batch_size=100), + "click_options": [ + *multiprocessing_options(default_max_batch_size=100), + click.Option( + ["--flusher-processes", "flusher_processes"], + default=1, + type=int, + help="Maximum number of processes for the span flusher. Defaults to 1.", + ), + ], }, "process-segments": { "topic": Topic.BUFFERED_SEGMENTS, @@ -38,6 +38,7 @@ class ProcessSpansStrategyFactory(ProcessingStrategyFactory[KafkaPayload]): num_processes: int, input_block_size: int | None, output_block_size: int | None, + flusher_processes: int | None = None, produce_to_pipe: Callable[[KafkaPayload], None] | None = None, ): super().__init__() @@ -48,6 +49,7 @@ class ProcessSpansStrategyFactory(ProcessingStrategyFactory[KafkaPayload]): self.input_block_size = input_block_size self.output_block_size = output_block_size self.num_processes = num_processes + self.flusher_processes = flusher_processes self.produce_to_pipe = produce_to_pipe if self.num_processes != 1: @@ -69,6 +71,7 @@ class ProcessSpansStrategyFactory(ProcessingStrategyFactory[KafkaPayload]): flusher = self._flusher = SpanFlusher( buffer, next_step=committer, + max_processes=self.flusher_processes, produce_to_pipe=self.produce_to_pipe, ) @@ -15,6 +15,7 @@ from arroyo.types import FilteredPayload, Message from sentry import options from sentry.conf.types.kafka_definition import Topic +from sentry.processing.backpressure.memory import ServiceMemory from sentry.spans.buffer import SpansBuffer from sentry.utils import metrics from sentry.utils.arroyo import run_with_initialized_sentry @@ -27,7 +28,8 @@ logger = logging.getLogger(__name__) class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): """ - A background thread that polls Redis for new segments to flush and to produce to Kafka. + A background multiprocessing manager that polls Redis for new segments to flush and to produce to Kafka. + Creates one process per shard for parallel processing. This is a processing step to be embedded into the consumer that writes to Redis. It takes and fowards integer messages that represent recently @@ -42,27 +44,53 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): self, buffer: SpansBuffer, next_step: ProcessingStrategy[FilteredPayload | int], + max_processes: int | None = None, produce_to_pipe: Callable[[KafkaPayload], None] | None = None, ): - self.buffer = buffer self.next_step = next_step + self.max_processes = max_processes or len(buffer.assigned_shards) self.mp_context = mp_context = multiprocessing.get_context("spawn") self.stopped = mp_context.Value("i", 0) self.redis_was_full = False self.current_drift = mp_context.Value("i", 0) - self.backpressure_since = mp_context.Value("i", 0) - self.healthy_since = mp_context.Value("i", 0) - self.process_restarts = 0 self.produce_to_pipe = produce_to_pipe - self._create_process() - - def _create_process(self): + # Determine which shards get their own processes vs shared processes + self.num_processes = min(self.max_processes, len(buffer.assigned_shards)) + self.process_to_shards_map: dict[int, list[int]] = { + i: [] for i in range(self.num_processes) + } + for i, shard in enumerate(buffer.assigned_shards): + process_index = i % self.num_processes + self.process_to_shards_map[process_index].append(shard) + + self.processes: dict[int, multiprocessing.context.SpawnProcess | threading.Thread] = {} + self.process_healthy_since = { + process_index: mp_context.Value("i", int(time.time())) + for process_index in range(self.num_processes) + } + self.process_backpressure_since = { + process_index: mp_context.Value("i", 0) for process_index in range(self.num_processes) + } + self.process_restarts = {process_index: 0 for process_index in range(self.num_processes)} + self.buffers: dict[int, SpansBuffer] = {} + + self._create_processes() + + def _create_processes(self): + # Create processes based on shard mapping + for process_index, shards in self.process_to_shards_map.items(): + self._create_process_for_shards(process_index, shards) + + def _create_process_for_shards(self, process_index: int, shards: list[int]): # Optimistically reset healthy_since to avoid a race between the # starting process and the next flush cycle. Keep back pressure across # the restart, however. - self.healthy_since.value = int(time.time()) + self.process_healthy_since[process_index].value = int(time.time()) + + # Create a buffer for these specific shards + shard_buffer = SpansBuffer(shards) make_process: Callable[..., multiprocessing.context.SpawnProcess | threading.Thread] if self.produce_to_pipe is None: @@ -72,37 +100,50 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): # pickled separately. at the same time, pickling # synchronization primitives like multiprocessing.Value can # only be done by the Process - self.buffer, + shard_buffer, ) make_process = self.mp_context.Process else: - target = partial(SpanFlusher.main, self.buffer) + target = partial(SpanFlusher.main, shard_buffer) make_process = threading.Thread - self.process = make_process( + process = make_process( target=target, args=( + shards, self.stopped, self.current_drift, - self.backpressure_since, - self.healthy_since, + self.process_backpressure_since[process_index], + self.process_healthy_since[process_index], self.produce_to_pipe, ), daemon=True, ) - self.process.start() + process.start() + self.processes[process_index] = process + self.buffers[process_index] = shard_buffer + + def _create_process_for_shard(self, shard: int): + # Find which process this shard belongs to and restart that process + for process_index, shards in self.process_to_shards_map.items(): + if shard in shards: + self._create_process_for_shards(process_index, shards) + break @staticmethod def main( buffer: SpansBuffer, + shards: list[int], stopped, current_drift, backpressure_since, healthy_since, produce_to_pipe: Callable[[KafkaPayload], None] | None, ) -> None: + shard_tag = ",".join(map(str, shards)) sentry_sdk.set_tag("sentry_spans_buffer_component", "flusher") + sentry_sdk.set_tag("sentry_spans_buffer_shards", shard_tag) try: producer_futures = [] @@ -134,23 +175,28 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): else: backpressure_since.value = 0 + # Update healthy_since for all shards handled by this process healthy_since.value = system_now if not flushed_segments: time.sleep(1) continue - with metrics.timer("spans.buffer.flusher.produce"): - for _, flushed_segment in flushed_segments.items(): + with metrics.timer("spans.buffer.flusher.produce", tags={"shard": shard_tag}): + for flushed_segment in flushed_segments.values(): if not flushed_segment.spans: continue spans = [span.payload for span in flushed_segment.spans] kafka_payload = KafkaPayload(None, orjson.dumps({"spans": spans}), []) - metrics.timing("spans.buffer.segment_size_bytes", len(kafka_payload.value)) + metrics.timing( + "spans.buffer.segment_size_bytes", + len(kafka_payload.value), + tags={"shard": shard_tag}, + ) produce(kafka_payload) - with metrics.timer("spans.buffer.flusher.wait_produce"): + with metrics.timer("spans.buffer.flusher.wait_produce", tags={"shards": shard_tag}): for future in producer_futures: future.result() @@ -169,27 +215,48 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): def poll(self) -> None: self.next_step.poll() - def _ensure_process_alive(self) -> None: + def _ensure_processes_alive(self) -> None: max_unhealthy_seconds = options.get("spans.buffer.flusher.max-unhealthy-seconds") - if not self.process.is_alive(): - exitcode = getattr(self.process, "exitcode", "unknown") - cause = f"no_process_{exitcode}" - elif int(time.time()) - self.healthy_since.value > max_unhealthy_seconds: - cause = "hang" - else: - return # healthy - metrics.incr("spans.buffer.flusher_unhealthy", tags={"cause": cause}) - if self.process_restarts > MAX_PROCESS_RESTARTS: - raise RuntimeError(f"flusher process crashed repeatedly ({cause}), restarting consumer") + for process_index, process in self.processes.items(): + if not process: + continue + + shards = self.process_to_shards_map[process_index] + + cause = None + if not process.is_alive(): + exitcode = getattr(process, "exitcode", "unknown") + cause = f"no_process_{exitcode}" + elif ( + int(time.time()) - self.process_healthy_since[process_index].value + > max_unhealthy_seconds + ): + # Check if any shard handled by this process is unhealthy + cause = "hang" + + if cause is None: + continue # healthy + + # Report unhealthy for all shards handled by this process + for shard in shards: + metrics.incr( + "spans.buffer.flusher_unhealthy", tags={"cause": cause, "shard": shard} + ) - try: - self.process.kill() - except ValueError: - pass # Process already closed, ignore + if self.process_restarts[process_index] > MAX_PROCESS_RESTARTS: + raise RuntimeError( + f"flusher process for shards {shards} crashed repeatedly ({cause}), restarting consumer" + ) + self.process_restarts[process_index] += 1 - self.process_restarts += 1 - self._create_process() + try: + if isinstance(process, multiprocessing.Process): + process.kill() + except (ValueError, AttributeError): + pass # Process already closed, ignore + + self._create_process_for_shards(process_index, shards) def submit(self, message: Message[FilteredPayload | int]) -> None: # Note that submit is not actually a hot path. Their message payloads @@ -197,18 +264,22 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): # per second at most. If anything, self.poll() might even be called # more often than submit() - self._ensure_process_alive() + self._ensure_processes_alive() - self.buffer.record_stored_segments() + for buffer in self.buffers.values(): + buffer.record_stored_segments() # We pause insertion into Redis if the flusher is not making progress # fast enough. We could backlog into Redis, but we assume, despite best # efforts, it is still always going to be less durable than Kafka. # Minimizing our Redis memory usage also makes COGS easier to reason # about. - if self.backpressure_since.value > 0: - backpressure_secs = options.get("spans.buffer.flusher.backpressure-seconds") - if int(time.time()) - self.backpressure_since.value > backpressure_secs: + backpressure_secs = options.get("spans.buffer.flusher.backpressure-seconds") + for backpressure_since in self.process_backpressure_since.values(): + if ( + backpressure_since.value > 0 + and int(time.time()) - backpressure_since.value > backpressure_secs + ): metrics.incr("spans.buffer.flusher.backpressure") raise MessageRejected() @@ -225,7 +296,9 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): # wait until the situation is improved manually. max_memory_percentage = options.get("spans.buffer.max-memory-percentage") if max_memory_percentage < 1.0: - memory_infos = list(self.buffer.get_memory_info()) + memory_infos: list[ServiceMemory] = [] + for buffer in self.buffers.values(): + memory_infos.extend(buffer.get_memory_info()) used = sum(x.used for x in memory_infos) available = sum(x.available for x in memory_infos) if available > 0 and used / available > max_memory_percentage: @@ -253,15 +326,22 @@ class SpanFlusher(ProcessingStrategy[FilteredPayload | int]): self.next_step.close() def join(self, timeout: float | None = None): - # set stopped flag first so we can "flush" the background thread while + # set stopped flag first so we can "flush" the background threads while # next_step is also shutting down. we can do two things at once! self.stopped.value = True deadline = time.time() + timeout if timeout else None self.next_step.join(timeout) - while self.process.is_alive() and (deadline is None or deadline > time.time()): - time.sleep(0.1) + # Wait for all processes to finish + for process_index, process in self.processes.items(): + if deadline is not None: + remaining_time = deadline - time.time() + if remaining_time <= 0: + break + + while process.is_alive() and (deadline is None or deadline > time.time()): + time.sleep(0.1) - if isinstance(self.process, multiprocessing.Process): - self.process.terminate() + if isinstance(process, multiprocessing.Process): + process.terminate() @@ -1,3 +1,4 @@ +import time from datetime import datetime import pytest @@ -8,7 +9,7 @@ from arroyo.types import BrokerValue, Message, Partition, Topic from sentry.spans.consumers.process.factory import ProcessSpansStrategyFactory -@pytest.mark.django_db +@pytest.mark.django_db(transaction=True) def test_basic(monkeypatch): # Flush very aggressively to make test pass instantly monkeypatch.setattr("time.sleep", lambda _: None) @@ -56,6 +57,10 @@ def test_basic(monkeypatch): step.poll() fac._flusher.current_drift.value = 9000 # "advance" our "clock" + step.poll() + # Give flusher threads time to process after drift change + time.sleep(0.1) + step.join() (msg,) = messages @@ -74,3 +79,45 @@ def test_basic(monkeypatch): }, ], } + + +@pytest.mark.django_db(transaction=True) +def test_flusher_processes_limit(monkeypatch): + """Test that flusher respects the max_processes limit""" + # Flush very aggressively to make test pass instantly + monkeypatch.setattr("time.sleep", lambda _: None) + + topic = Topic("test") + messages: list[KafkaPayload] = [] + + # Create factory with limited flusher processes + fac = ProcessSpansStrategyFactory( + max_batch_size=10, + max_batch_time=10, + num_processes=1, + input_block_size=None, + output_block_size=None, + flusher_processes=2, # Limit to 2 processes even if more shards + produce_to_pipe=messages.append, + ) + + commits = [] + + def add_commit(offsets, force=False): + commits.append(offsets) + + # Create with 4 partitions/shards to test process sharing + partitions = {Partition(topic, i): 0 for i in range(4)} + step = fac.create_with_partitions(add_commit, partitions) + + # Verify that flusher uses at most 2 processes + flusher = fac._flusher + assert len(flusher.processes) == 2 + assert flusher.max_processes == 2 + assert flusher.num_processes == 2 + + # Verify shards are distributed across processes + total_shards = sum(len(shards) for shards in flusher.process_to_shards_map.values()) + assert total_shards == 4 # All 4 shards should be assigned + + step.join() @@ -80,4 +80,4 @@ def test_backpressure(monkeypatch): assert messages - assert flusher.backpressure_since.value + assert any(x.value for x in flusher.process_backpressure_since.values())
Leaderboard for this case
hard · 2 of 9 bots fully caught it (22% solve rate) · 50% avg coverage.
| # | Bot | Result | Caught | Coverage | Findings | False alarms |
|---|---|---|---|---|---|---|
| 1 | Claude Code (Opus 5.5) | solved | 2/2 | 100.0% | 6 | 0 |
| 2 | Mjolnir (Sonnet 5.5) | solved | 2/2 | 100.0% | 3 | 0 |
| 3 | cursor | partial | 1/2 | 80.0% | 1 | 0 |
| 4 | Drape (Deep Review) | partial | 1/2 | 80.0% | 5 | 0 |
| 5 | Drape (Review) | partial | 1/2 | 80.0% | 5 | 0 |
| 6 | Mjolnir (Opus 5.5) | partial | 1/2 | 80.0% | 2 | 0 |
| 7 | Cubic | partial | 1/2 | 20.0% | 5 | 0 |
| 8 | greptile | miss | 0/2 | 0.0% | 2 | 0 |
| 9 | qodo | miss | 0/2 | 0.0% | 0 | 0 |
Bot results
Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | exact |
| d5 | low | ✓ | off by one |
flush_segments computes max_segments_per_shard = ceil(max_flush_segments / len(self.assigned_shards)). Per-process buffers have fewer shards, so the total per cycle becomes roughly N times the option. get_memory_info now runs iter_cluster_memory_usage once per buffer against the same cluster, so the results are duplicated. [grounded: diff] [consensus 3/3: valid=3]
wait_produce uses the tag key 'shards' while the sibling metrics use 'shard', and the value is a comma-joined list of shard IDs. Both come from this change. [grounded: diff] [consensus 3/3: valid=3]
The new method is never called, and Thread/Process objects are always truthy, so the `if not process` guard never fires. This is dead code, not a behaviour bug. [grounded: diff] [consensus 3/3: minor=3]
The test patches time.sleep on the time module and then calls time.sleep(0.1), so the added wait does nothing and the test still races with the flusher thread. [grounded: tree tests/sentry/spans/consumers/process/test_consumer.py] [consensus 3/3: valid=3]
raw comments (6)
The restart path calls `process.kill()` only when `isinstance(process, multiprocessing.Process)` is true. Processes created via `get_context("spawn").Process` are `SpawnProcess` instances, and those are NOT instances of `multiprocessing.Process`. I checked this and the isinstance call returns False. The old code called `self.process.kill()` unconditionally, so this is a regression: a hung flusher process ('hang' cause) is never killed. A replacement process is then started for the same shards, while the old one keeps running and flushing the same Redis queues. Each hang adds another orphaned process, which leaks processes and causes concurrent or duplicate flushing of segments. The check should be against `multiprocessing.process.BaseProcess`, or it should test `hasattr(process, 'kill')` / not a Thread.
In `join()`, the loop `break`s as soon as the deadline has passed. Every remaining process then skips both the wait and the `terminate()` call, so later flusher processes are never stopped explicitly. Separately, the `isinstance(process, multiprocessing.Process)` guard is always False for `SpawnProcess`, so `terminate()` never runs for any process. With multiple processes, shutdown can leave flusher processes running until interpreter exit.
Each process now gets its own `SpansBuffer(shards)`, and `flush_segments` gives every buffer the full `spans.buffer.max-flush-segments` budget. As a result, the effective per-cycle flush limit is multiplied by the number of flusher processes. This silently changes the meaning of that operational option. Also, `get_memory_info()` is now called once per buffer against the same Redis cluster, so every submit makes N redundant cluster INFO calls, and the `used`/`available` lists are N-fold duplicates.
The tag keys are inconsistent. `spans.buffer.flusher.wait_produce` uses the tag `shards`, while `produce` and `segment_size_bytes` use `shard`. Also, a comma-joined list of shard IDs is used as a tag value, which raises metric cardinality.
`_create_process_for_shard` is dead code that is never called. Also, `if not process: continue` in `_ensure_processes_alive` is unreachable.
`time.sleep(0.1)` was added to give flusher threads time to run, but `time.sleep` is monkeypatched to a no-op earlier in this test. The sleep does nothing, so the test still depends on timing and may be flaky.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✗ | — |
| d5 | low | ✓ | off by one |
A value of 0 falls back to the shard count through `or`. A negative value is truthy, though, so num_processes becomes negative, range() returns no keys, and `process_to_shards_map[0]` raises a KeyError. The option type=int doesn't prevent this. [grounded: tree src/sentry/spans/consumers/process/flusher.py] [consensus 5/5: valid=5]
time.sleep is monkeypatched on the time module, so the added `time.sleep(0.1)` does nothing and the flusher thread still isn't synchronized. [grounded: tree tests/sentry/spans/consumers/process/test_consumer.py] [consensus 5/5: valid=5]
get_memory_info calls iter_cluster_memory_usage(self.client), a cluster-wide query. Looping over every per-process buffer repeats it N times on each submit. [grounded: tree src/sentry/spans/consumers/process/flusher.py] [consensus 5/5: valid=5]
The helper really is unused, but this is dead-code cleanup, not a behavior defect. [grounded: diff] [consensus 5/5: minor=5]
raw comments (6)
<!-- cubic:v=314d9919-fc39-4114-9824-0b393af66ecf -->
<!-- cubic:review-run=2a961d26-e8df-597d-baa9-0402fb76cf06 -->
<!-- metadata:{"confidence":10} -->
P2: When `--flusher-processes` is zero or negative, this option accepts it and the flusher cannot honor it safely. Reject values below 1 with `click.IntRange(min=1)`.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At src/sentry/consumers/__init__.py, line 435:
<comment>When `--flusher-processes` is zero or negative, this option accepts it and the flusher cannot honor it safely. Reject values below 1 with `click.IntRange(min=1)`.</comment>
<file context>
@@ -427,7 +427,15 @@ def ingest_transactions_options() -> list[click.Option]:
+ click.Option(
+ ["--flusher-processes", "flusher_processes"],
+ default=1,
+ type=int,
+ help="Maximum number of processes for the span flusher. Defaults to 1.",
+ ),
</file context>
```
</details>
```suggestion
type=click.IntRange(min=1),
```
<!-- cubic:v=d7b11edd-a65b-4a0b-910f-94bf6971fff0 -->
<!-- cubic:review-run=2a961d26-e8df-597d-baa9-0402fb76cf06 -->
<!-- metadata:{"confidence":9} -->
P2: In test_basic, time.sleep(0.1) is a no-op: line 15 already monkeypatched time.sleep to lambda _: None, and the flusher thread calls the same patched time.sleep. So this added sleep gives the flusher no time to process the drift change and the test stays racy/timing-dependent. Remove the patch for this sleep (e.g. capture the real time.sleep and call it), or otherwise synchronize with the flusher.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At tests/sentry/spans/consumers/process/test_consumer.py, line 62:
<comment>In test_basic, time.sleep(0.1) is a no-op: line 15 already monkeypatched time.sleep to lambda _: None, and the flusher thread calls the same patched time.sleep. So this added sleep gives the flusher no time to process the drift change and the test stays racy/timing-dependent. Remove the patch for this sleep (e.g. capture the real time.sleep and call it), or otherwise synchronize with the flusher.</comment>
<file context>
@@ -56,6 +57,10 @@ def add_commit(offsets, force=False):
+ step.poll()
+ # Give flusher threads time to process after drift change
+ time.sleep(0.1)
+
step.join()
</file context>
```
</details>
<!-- cubic:v=77a978de-61c3-473e-b1aa-9f8014e50343 -->
<!-- cubic:review-run=2a961d26-e8df-597d-baa9-0402fb76cf06 -->
<!-- metadata:{"confidence":9} -->
P2: When hard memory backpressure is enabled, this repeats cluster-wide Redis memory queries once per flusher process. Query memory from one buffer once instead of iterating over every process buffer.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At src/sentry/spans/consumers/process/flusher.py, line 301:
<comment>When hard memory backpressure is enabled, this repeats cluster-wide Redis memory queries once per flusher process. Query memory from one buffer once instead of iterating over every process buffer.</comment>
<file context>
@@ -225,7 +296,9 @@ def submit(self, message: Message[FilteredPayload | int]) -> None:
- memory_infos = list(self.buffer.get_memory_info())
+ memory_infos: list[ServiceMemory] = []
+ for buffer in self.buffers.values():
+ memory_infos.extend(buffer.get_memory_info())
used = sum(x.used for x in memory_infos)
available = sum(x.available for x in memory_infos)
</file context>
```
</details>
<!-- cubic:v=18a9bda4-4d92-42fb-b7e1-84d21dbeb1c5 -->
<!-- cubic:review-run=2a961d26-e8df-597d-baa9-0402fb76cf06 -->
<!-- metadata:{"confidence":9} -->
P2: When `next_step.join(timeout)` consumes the deadline, this `break` skips termination for every remaining flusher process. Continue through the loop without waiting after the deadline so each process reaches `terminate()`.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At src/sentry/spans/consumers/process/flusher.py, line 132:
<comment>When `next_step.join(timeout)` consumes the deadline, this `break` skips termination for every remaining flusher process. Continue through the loop without waiting after the deadline so each process reaches `terminate()`.</comment>
<file context>
@@ -72,37 +100,50 @@ def _create_process(self):
+ for process_index, shards in self.process_to_shards_map.items():
+ if shard in shards:
+ self._create_process_for_shards(process_index, shards)
+ break
@staticmethod
</file context>
```
</details>
```suggestion
pass
```
<!-- cubic:v=fc809c6d-ac4a-435c-9c30-05db881d7822 -->
<!-- cubic:review-run=2a961d26-e8df-597d-baa9-0402fb76cf06 -->
<!-- metadata:{"confidence":10} -->
P3: This helper is never called because the restart path invokes `_create_process_for_shards` directly. Remove the unused helper to avoid maintaining a misleading shard-restart path.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At src/sentry/spans/consumers/process/flusher.py, line 127:
<comment>This helper is never called because the restart path invokes `_create_process_for_shards` directly. Remove the unused helper to avoid maintaining a misleading shard-restart path.</comment>
<file context>
@@ -72,37 +100,50 @@ def _create_process(self):
+ self.processes[process_index] = process
+ self.buffers[process_index] = shard_buffer
+
+ def _create_process_for_shard(self, shard: int):
+ # Find which process this shard belongs to and restart that process
+ for process_index, shards in self.process_to_shards_map.items():
</file context>
```
</details>
<!-- cubic:review-summary:start -->
**5 issues found** across 6 files
<!-- cubic:review-summary:end -->
<details>
<summary>Prompt for AI agents (unresolved issues)</summary>
```text
Check if these issues are valid — if so, understand the root cause of each and fix them. If appropriate, use sub-agents to investigate and fix each issue separately.
<file name="src/sentry/consumers/__init__.py">
<violation number="1" location="src/sentry/consumers/__init__.py:435">
P2: When `--flusher-processes` is zero or negative, this option accepts it and the flusher cannot honor it safely. Reject values below 1 with `click.IntRange(min=1)`.</violation>
</file>
<file name="tests/sentry/spans/consumers/process/test_consumer.py">
<violation number="1" location="tests/sentry/spans/consumers/process/test_consumer.py:62">
P2: In test_basic, time.sleep(0.1) is a no-op: line 15 already monkeypatched time.sleep to lambda _: None, and the flusher thread calls the same patched time.sleep. So this added sleep gives the flusher no time to process the drift change and the test stays racy/timing-dependent. Remove the patch for this sleep (e.g. capture the real time.sleep and call it), or otherwise synchronize with the flusher.</violation>
</file>
<file name="src/sentry/spans/consumers/process/flusher.py">
<violation number="1" location="src/sentry/spans/consumers/process/flusher.py:127">
P3: This helper is never called because the restart path invokes `_create_process_for_shards` directly. Remove the unused helper to avoid maintaining a misleading shard-restart path.</violation>
<violation number="2" location="src/sentry/spans/consumers/process/flusher.py:132">
P2: When `next_step.join(timeout)` consumes the deadline, this `break` skips termination for every remaining flusher process. Continue through the loop without waiting after the deadline so each process reaches `terminate()`.</violation>
<violation number="3" location="src/sentry/spans/consumers/process/flusher.py:301">
P2: When hard memory backpressure is enabled, this repeats cluster-wide Redis memory queries once per flusher process. Query memory from one buffer once instead of iterating over every process buffer.</violation>
</file>
```
</details>
<sub>Reply with feedback, questions, or to request a fix.<br /><br />[Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/75/ai_pr_review_github_delivery_0500034514f5a920579c751737231b4dac760afbe3f90b50bd8cf7a81c319c5e?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F75)</sub>
<!-- cubic:review-post:ai_pr_review_github_delivery_0500034514f5a920579c751737231b4dac760afbe3f90b50bd8cf7a81c319c5e:9d3326c5d902e1e263c2b610a9e75716ec9d81ce:2a961d26-e8df-597d-baa9-0402fb76cf06 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | exact |
| d5 | low | ✗ | — |
raw comments (2)
### Hung flushers keep running
**High Severity**
<!-- DESCRIPTION START -->
`isinstance` against `multiprocessing.Process` does not match spawn-context workers, so `kill` is skipped on hang. A replacement starts while the original keeps flushing the same shards, which can duplicate segments on `buffered-segments`.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: 55395e90-2154-4d2e-9c0e-62dd1094716d -->
<!-- LOCATIONS START
src/sentry/spans/consumers/process/flusher.py#L252-L257
src/sentry/spans/consumers/process/flusher.py#L345-L347
LOCATIONS END -->
<details>
<summary>Additional Locations (1)</summary>
- [`src/sentry/spans/consumers/process/flusher.py#L345-L347`](https://github.com/drape-io/crbench-cursor-2026-09/blob/f56e538304007a49640e0b79a00325f71d462f79/src/sentry/spans/consumers/process/flusher.py#L345-L347)
</details>
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmUzYjNiMTVmLTBjZDgtNDZkZS04NDczLWNhMGI3NGM3ODJmZSIsImVuY3J5cHRpb25LZXkiOiJJSXUwb2xHd050Q3FSanF5NkV3YW51bW9RcXFOM3JFd1hwc0Vuem1iWi1NIiwiYnJhbmNoIjoidDItc2VudHJ5LTkzODI0IiwicmVwb093bmVyIjoiZHJhcGUtaW8iLCJyZXBvTmFtZSI6ImNyYmVuY2gtY3Vyc29yLTIwMjYtMDkiLCJwcm92aWRlciI6ImdpdGh1YiJ9fQ" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmUzYjNiMTVmLTBjZDgtNDZkZS04NDczLWNhMGI3NGM3ODJmZSIsImVuY3J5cHRpb25LZXkiOiJJSXUwb2xHd050Q3FSanF5NkV3YW51bW9RcXFOM3JFd1hwc0Vuem1iWi1NIiwiYnJhbmNoIjoidDItc2VudHJ5LTkzODI0IiwicmVwb093bmVyIjoiZHJhcGUtaW8iLCJyZXBvTmFtZSI6ImNyYmVuY2gtY3Vyc29yLTIwMjYtMDkiLCJwck51bWJlciI6OTYsImNvbW1pdFNoYSI6ImY1NmU1MzgzMDQwMDdhNDk2NDBlMGI3OWEwMDMyNWY3MWQ0NjJmNzkiLCJwcm92aWRlciI6ImdpdGh1YiJ9fQ" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit f56e538304007a49640e0b79a00325f71d462f79. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_REVIEW -->
Cursor Bugbot has reviewed your changes using high effort and found 1 potential issue.
<!-- BUGBOT_FIX_ALL -->
<a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aTo2ZGM1NWFjOC0zZWJhLTQwNWYtYWRkOC1jZGFkZTkyZWI1MWUiLCJlbmNyeXB0aW9uS2V5IjoiR1A1aFNlcjFZUTFnOEFHRlczZ2hlcXZoc3dSbmx0c1dNcDREUU5XMS1mWSIsImJyYW5jaCI6InQyLXNlbnRyeS05MzgyNCIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>
<!-- /BUGBOT_FIX_ALL -->
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN -->
<sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END -->
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit f56e538304007a49640e0b79a00325f71d462f79. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | exact |
| d5 | low | ✗ | — |
Before the change, a single buffer divided max-flush-segments across all shards. Now each per-process SpansBuffer divides it only across the shards it owns, so every process can flush the full limit and the per-shard at-limit threshold goes up. [grounded: tree src/sentry/spans/buffer.py] [consensus 3/3: valid=3]
With a negative value, range(num_processes) is empty but i % -1 is 0, so the map lookup raises KeyError. A value of 0 falls through to len(shards) because of 'or'. Nothing validates the option. [grounded: diff] [consensus 3/3: valid=3]
The new wait_produce metric uses the tag key 'shards', while the sibling metrics added in this change use 'shard'. The inconsistency is real but only affects observability. [grounded: diff] [consensus 3/3: valid=3]
Every buffer uses the same span buffer Redis cluster client, so each submit now queries cluster memory N times and counts each node N times. The used/available ratio is unchanged, but the redundant queries are a real, if small, performance cost. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
**max-flush-segments limit multiplied by process count**
Each per-process `SpansBuffer` only sees its own shards, so `flush_segments` divides `spans.buffer.max-flush-segments` by fewer shards. Each process can then flush the full limit, and the at-limit backpressure check uses a larger per-shard threshold. Divide the flush budget by the total shard count, not by the shards each process owns.
```
src/sentry/spans/consumers/process/flusher.py:93 shard_buffer = SpansBuffer(shards)
src/sentry/spans/consumers/process/flusher.py:59 # Determine which shards get their own processes vs shared processes
src/sentry/spans/consumers/process/flusher.py:60 self.num_processes = min(self.max_processes, len(buffer.assigned_shards))
src/sentry/spans/consumers/process/flusher.py:61 self.process_to_shards_map: dict[int, list[int]] = {
src/sentry/spans/buffer.py:382 shard_factor = max(1, len(self.assigned_shards))
src/sentry/spans/buffer.py:383 max_flush_segments = options.get("spans.buffer.max-flush-segments")
src/sentry/spans/buffer.py:384 max_segments_per_shard = math.ceil(max_flush_segments / shard_factor)
src/sentry/spans/buffer.py:411 if len(segment) >= max_segments_per_shard:
src/sentry/spans/buffer.py:412 any_shard_at_limit = True
```
**Non-positive --flusher-processes crashes or is ignored**
`max_processes` is never checked. 0 quietly turns into one process per shard. A negative value makes `range(num_processes)` empty, so building the shard map raises a KeyError. Use `click.IntRange(min=1)` for the option and validate `max_processes >= 1`.
```
src/sentry/spans/consumers/process/flusher.py:51 self.max_processes = max_processes or len(buffer.assigned_shards)
src/sentry/spans/consumers/process/flusher.py:60 self.num_processes = min(self.max_processes, len(buffer.assigned_shards))
src/sentry/spans/consumers/process/flusher.py:61 self.process_to_shards_map: dict[int, list[int]] = {
src/sentry/spans/consumers/process/flusher.py:62 i: [] for i in range(self.num_processes)
src/sentry/spans/consumers/process/flusher.py:64 for i, shard in enumerate(buffer.assigned_shards):
src/sentry/spans/consumers/process/flusher.py:65 process_index = i % self.num_processes
src/sentry/spans/consumers/process/flusher.py:66 self.process_to_shards_map[process_index].append(shard)
```
**Unhealthy flusher processes are never killed on restart**
`isinstance(process, multiprocessing.Process)` is always False for processes from the spawn context (`SpawnProcess` doesn't subclass `multiprocessing.Process`), so `kill()` never runs. A hung flusher keeps running next to its replacement, and both flush the same shards. Call `kill()` whenever the object has it (`if not isinstance(process, threading.Thread)`), as the old code did.
```
src/sentry/spans/consumers/process/flusher.py:253 try:
src/sentry/spans/consumers/process/flusher.py:254 if isinstance(process, multiprocessing.Process):
src/sentry/spans/consumers/process/flusher.py:255 process.kill()
src/sentry/spans/consumers/process/flusher.py:256 except (ValueError, AttributeError):
src/sentry/spans/consumers/process/flusher.py:53 self.mp_context = mp_context = multiprocessing.get_context("spawn")
src/sentry/spans/consumers/process/flusher.py:105 make_process = self.mp_context.Process
src/sentry/spans/consumers/process/flusher.py:198 (base) # more often than submit()
src/sentry/spans/consumers/process/flusher.py:200 (base) self._ensure_process_alive()
```
**wait_produce metric tagged 'shards' instead of 'shard'**
On every flush cycle, `spans.buffer.flusher.wait_produce` is emitted with tag key `shards`, while the other new flusher metrics use `shard`, so filtering on `shard` drops this timer. Rename the tag key to `shard`.
```
src/sentry/spans/consumers/process/flusher.py:185 with metrics.timer("spans.buffer.flusher.produce", tags={"shard": shard_tag}):
src/sentry/spans/consumers/process/flusher.py:192 metrics.timing(
src/sentry/spans/consumers/process/flusher.py:193 "spans.buffer.segment_size_bytes",
src/sentry/spans/consumers/process/flusher.py:194 len(kafka_payload.value),
src/sentry/spans/consumers/process/flusher.py:144 shard_tag = ",".join(map(str, shards))
src/sentry/spans/consumers/process/flusher.py:199 with metrics.timer("spans.buffer.flusher.wait_produce", tags={"shards": shard_tag}):
```
**Redis memory info queried once per flusher process**
Every buffer uses the same Redis cluster, so calling `get_memory_info()` on each buffer queries all cluster nodes N times on every submit and counts each node N times. Call `get_memory_info()` on a single buffer.
```
src/sentry/spans/consumers/process/flusher.py:299 memory_infos: list[ServiceMemory] = []
src/sentry/spans/consumers/process/flusher.py:300 for buffer in self.buffers.values():
src/sentry/spans/consumers/process/flusher.py:301 memory_infos.extend(buffer.get_memory_info())
src/sentry/spans/buffer.py:375 def get_memory_info(self) -> Generator[ServiceMemory]:
src/sentry/spans/buffer.py:376 return iter_cluster_memory_usage(self.client)
```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | exact |
| d5 | low | ✗ | — |
The docstring is inaccurate: shards are distributed with i % num_processes. This is a documentation issue, not a behavior bug. [grounded: diff] [consensus 3/3: minor=3]
get_memory_info calls iter_cluster_memory_usage(self.client), and every per-process buffer queries the same cluster. Each submit therefore runs N redundant cluster scans, which is a performance regression this change introduced. [grounded: diff] [consensus 3/3: valid=3]
The tag key differs between metrics (`shards` vs `shard`). This is a naming-consistency concern rather than a behavior bug. [grounded: diff] [consensus 3/3: minor=3]
A negative value is truthy, so num_processes becomes min(negative, n), which is negative. range() of that is empty, and i % -1 gives 0, so self.process_to_shards_map[0] raises KeyError. The new CLI option has no validation. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
**Hung flusher process is never killed before restart**
The kill is now guarded by `isinstance(process, multiprocessing.Process)`, but processes come from `mp_context.Process` (a `SpawnProcess`), which is not a subclass of `multiprocessing.Process`. The check is always False, so the old process is never killed. Call `process.kill()` unless it is a `threading.Thread`, or check against `multiprocessing.process.BaseProcess`.
```
src/sentry/spans/consumers/process/flusher.py:255 process.kill()
src/sentry/spans/consumers/process/flusher.py:256 except (ValueError, AttributeError):
src/sentry/spans/consumers/process/flusher.py:257 pass # Process already closed, ignore
src/sentry/spans/consumers/process/flusher.py:259 self._create_process_for_shards(process_index, shards)
src/sentry/spans/consumers/process/flusher.py:97 target = run_with_initialized_sentry(
src/sentry/spans/consumers/process/flusher.py:98 SpanFlusher.main,
src/sentry/spans/consumers/process/flusher.py:99 # unpickling buffer will import sentry, so it needs to be
src/sentry/spans/consumers/process/flusher.py:100 # pickled separately. at the same time, pickling
src/sentry/spans/consumers/process/flusher.py:101 # synchronization primitives like multiprocessing.Value can
src/sentry/spans/consumers/process/flusher.py:102 # only be done by the Process
```
**Docstring says one process per shard**
The class docstring says it "Creates one process per shard", but shards are grouped onto `max_processes` processes (`i % num_processes`). The CLI default of 1 puts all shards in a single process. Say shards are distributed across up to max_processes processes.
```
src/sentry/spans/consumers/process/flusher.py:31 A background multiprocessing manager that polls Redis for new segments to flush and to produce to Kafka.
src/sentry/spans/consumers/process/flusher.py:32 Creates one process per shard for parallel processing.
```
**Redis cluster INFO scanned N times per submit**
With N flusher processes, submit() calls get_memory_info() on every per-process buffer, and each call queries the whole Redis cluster. That is N identical cluster INFO scans per submit. Call get_memory_info() once, for example on the first buffer.
```
src/sentry/spans/consumers/process/flusher.py:296 # wait until the situation is improved manually.
src/sentry/spans/consumers/process/flusher.py:297 max_memory_percentage = options.get("spans.buffer.max-memory-percentage")
src/sentry/spans/consumers/process/flusher.py:298 if max_memory_percentage < 1.0:
src/sentry/spans/consumers/process/flusher.py:299 memory_infos: list[ServiceMemory] = []
src/sentry/spans/buffer.py:375 def get_memory_info(self) -> Generator[ServiceMemory]:
src/sentry/spans/buffer.py:376 return iter_cluster_memory_usage(self.client)
```
**Flusher timing metrics use inconsistent tag names**
The wait_produce timer is tagged `shards`, while the produce timer and segment_size metrics use `shard`. The tag value is also a comma-joined list of shards. Use the same tag key (`shard`) for all three metrics.
```
src/sentry/spans/consumers/process/flusher.py:185 with metrics.timer("spans.buffer.flusher.produce", tags={"shard": shard_tag}):
src/sentry/spans/consumers/process/flusher.py:199 with metrics.timer("spans.buffer.flusher.wait_produce", tags={"shards": shard_tag}):
```
**Negative flusher process count crashes consumer startup**
`--flusher-processes` is an unvalidated int. With a negative value, `min(self.max_processes, len(assigned_shards))` is negative, so the shard map is empty and the `i % self.num_processes` lookup raises KeyError when partitions are assigned. Validate the option (e.g. click.IntRange(min=1)) or clamp with max(1, ...).
```
src/sentry/spans/consumers/process/flusher.py:51 self.max_processes = max_processes or len(buffer.assigned_shards)
src/sentry/spans/consumers/process/flusher.py:60 self.num_processes = min(self.max_processes, len(buffer.assigned_shards))
src/sentry/spans/consumers/process/flusher.py:61 self.process_to_shards_map: dict[int, list[int]] = {
src/sentry/spans/consumers/process/flusher.py:62 i: [] for i in range(self.num_processes)
src/sentry/spans/consumers/process/flusher.py:63 }
src/sentry/spans/consumers/process/flusher.py:64 for i, shard in enumerate(buffer.assigned_shards):
src/sentry/spans/consumers/process/flusher.py:65 process_index = i % self.num_processes
src/sentry/spans/consumers/process/flusher.py:66 self.process_to_shards_map[process_index].append(shard)
```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✗ | — |
| d5 | low | ✗ | — |
The Click option has no range check. With 0, `0 or len(...)` falls back to the shard count. With -1, num_processes = min(-1, n) = -1, so range(-1) builds an empty map, and `process_to_shards_map[i % -1]` (index 0) raises KeyError. [grounded: diff] [consensus 3/3: valid=3]
The test calls monkeypatch.setattr("time.sleep", lambda _: None), which replaces the time module's sleep attribute, so the newly added `time.sleep(0.1)` does nothing. join() then sets stopped right away, and the flusher thread may exit before it flushes with the new drift. The assertion on the produced message is still racy. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Invalid Process Counts Accepted**
The new option accepts non-positive integers even though the flusher requires a positive process count. `--flusher-processes 0` is silently replaced with the shard count, which may launch more workers than configured. A negative value creates an empty process map and then raises `KeyError` while assigning the first shard. Please constrain the Click option or validate `max_processes` before constructing the map.
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Wait Is Still Disabled**
This delay does not wait for the background flusher because the test has already replaced the shared `time.sleep` function with a no-op. The worker runs asynchronously, and `join()` sets its stop flag before waiting, so the message assertion remains scheduler-dependent. Please synchronize on the produced message or avoid patching the sleep used for this wait.
<h3>Greptile Summary</h3>
This PR parallelizes span-buffer flushing by assigning shards across a configurable number of worker processes and tracking health, backpressure, restarts, buffers, and metrics per worker.
- Adds a `--flusher-processes` consumer option and passes it through the strategy factory.
- Splits assigned shards among flusher workers and independently restarts unhealthy workers.
- Updates shutdown, Redis-memory monitoring, and shard-tagged metrics for the multi-worker model.
- Extends consumer and flusher tests for the new process layout.
<h3>Confidence Score: 4/5</h3>
The PR appears safe to merge after non-blocking improvements to process-count validation and asynchronous test synchronization.
The multi-worker design has no established production-blocking failure, but non-positive CLI values behave incorrectly and the updated consumer test retains a timing race because its new wait is a no-op.
**Files Needing Attention:** src/sentry/consumers/__init__.py, tests/sentry/spans/consumers/process/test_consumer.py
<details><summary><h3>Important Files Changed</h3></summary>
| Filename | Overview |
|----------|----------|
| src/sentry/spans/consumers/process/flusher.py | Reworks the single span-flusher worker into a shard-distributed process manager with per-worker buffers, health state, backpressure, and restart handling. |
| src/sentry/consumers/__init__.py | Exposes the flusher process limit but does not constrain it to the positive values required by the process-mapping logic. |
| src/sentry/spans/consumers/process/factory.py | Threads the optional flusher process count from consumer construction into `SpanFlusher`. |
| tests/sentry/spans/consumers/process/test_consumer.py | Adds multi-worker distribution coverage, but its new asynchronous wait is disabled by the test's global sleep monkeypatch. |
| tests/sentry/spans/consumers/process/test_flusher.py | Updates the backpressure assertion for per-worker state. |
</details>
<details open><summary><h3>Flowchart</h3></summary>
```mermaid
%%{init: {'theme': 'neutral'}}%%
flowchart TD
C[process-spans consumer] --> F[ProcessSpansStrategyFactory]
F --> B[Assigned span-buffer shards]
F --> M[SpanFlusher manager]
M --> D{Distribute shards round-robin}
D --> W1[Worker 0 and shard buffer]
D --> W2[Worker 1 and shard buffer]
D --> WN[Worker N and shard buffer]
W1 --> R[(Redis)]
W2 --> R
WN --> R
W1 --> K[Kafka span output]
W2 --> K
WN --> K
M --> H[Per-worker health and backpressure checks]
H --> M
```
</details>
<!-- greptile_other_comments_section -->
<sub>Reviews (1): Last reviewed commit: ["Update CLAUDE.md and 5 more files"](https://github.com/drape-io/crbench-greptile-2026-09/commit/c22d61703066945bdf74ae616aed952062e437ad) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=64284830)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | exact |
| d5 | low | ✗ | — |
Production processes come from `multiprocessing.get_context("spawn").Process`, which creates a `SpawnProcess`. That class inherits from `BaseProcess`, not `multiprocessing.Process` (which is `context.Process`), so the new isinstance guard is always False and `kill()` is never called. The old code called `self.process.kill()` with no condition, so a hung process now keeps running while a replacement is started for the same shards. [grounded: diff] [consensus 3/3: valid=3]
raw comments (2)
[P1] Hung flusher processes are no longer killed before a replacement is spawned
The processes are created with `multiprocessing.get_context("spawn").Process`, which gives a `multiprocessing.context.SpawnProcess`. `SpawnProcess` subclasses `BaseProcess`, not `multiprocessing.Process`, so `isinstance(process, multiprocessing.Process)` is always False and `process.kill()` never runs (I checked: `isinstance(get_context('spawn').Process(...), multiprocessing.Process)` returns False). The old code called `self.process.kill()` unconditionally. Now, when a process is detected as a "hang" (still alive, but `healthy_since` is older than `max-unhealthy-seconds`), the stuck process keeps running. `_create_process_for_shards` then starts a second process for the same shards and overwrites the `self.processes[process_index]` entry. If the old process recovers, both flush the same Redis shards at the same time, which can produce duplicate segments. Each hang detection also leaks another orphaned process until the consumer exits. The check should match `SpawnProcess`/`BaseProcess`, or test for "not a `threading.Thread`" instead.
[P1] Unhealthy flusher processes are never killed before a replacement is spawned. In
, the old code called `process.kill()` unconditionally. The new code only calls it when `isinstance(process, multiprocessing.Process)` is true. In production the flusher processes are created with `multiprocessing.get_context("spawn").Process`, which returns a `SpawnProcess`. `SpawnProcess` subclasses `BaseProcess`, not `multiprocessing.Process`, so the check is always false (I confirmed that `isinstance(get_context('spawn').Process(), multiprocessing.Process)` returns `False`). When a process is detected as hung, it keeps running and `_create_process_for_shards` starts a second process for the same shards. Both then flush the same Redis queues at the same time, which can produce duplicate segments to Kafka. Each further restart, up to `MAX_PROCESS_RESTARTS`, leaks another live process
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✓ | off by one |
| d5 | low | ✓ | exact |
Processes come from `self.mp_context.Process`, which creates a `SpawnProcess`. `SpawnProcess` subclasses `BaseProcess`, not `multiprocessing.Process`, so this isinstance check is always False and `kill()` is never called. The removed code called `self.process.kill()` with no check, so a hung process is now left running next to its replacement on the same shards. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
[P2] Hung flusher process is no longer killed before restart
kill()` is now gated on `isinstance(process, multiprocessing.Process)`. The process is created from `mp_context.Process`, which is `SpawnProcess`. `SpawnProcess` subclasses `BaseProcess`, not `multiprocessing.Process`, so the check is always False in the real (non-pipe) path. The old code called `self.process.kill()` unconditionally. Now a flusher that is alive but hung (`cause == "hang"`) is never killed, and `_create_process_for_shards` starts a second flusher for the same shards. Hung processes pile up, and two flushers can flush the same shards concurrently. This only affects the hang path. The crashed-process path is unaffected because the process is already dead. The same wrong `isinstance` check in `join()` means `terminate()` is never called on shutdown, but that is carried over from the old code
[P1] Hung flusher processes are no longer killed on restart
The kill is now guarded by `isinstance(process, multiprocessing.Process)`, but the processes are created from the spawn context (`mp_context.Process`, a `SpawnProcess`). I checked that `issubclass(get_context('spawn').Process, multiprocessing.Process)` is False, so the guard never passes and `process.kill()` is never called. The old code killed unconditionally. When a flusher is detected as "hang" (alive but `healthy_since` stale), a replacement process is started for the same shards while the hung one keeps running. This leaks processes and can leave two flushers on the same shards. Each restart adds another orphan until `MAX_PROCESS_RESTARTS` is reached. Check against `multiprocessing.context.SpawnProcess` or `BaseProcess`, or just call `kill()` and catch `AttributeError
[P3] join() stops terminating processes once the deadline has passed
When `remaining_time ‹= 0` the loop `break`s. That skips `terminate()` for every process not yet visited, so those spawned children are left running. The same `isinstance(process, multiprocessing.Process)` guard is also False for `SpawnProcess`, so `terminate()` is never called at all. Use `continue` instead of `break`, and fix the type check
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d3 | high | ✗ | — |
| d5 | low | ✗ | — |
raw comments (2)
<h3>Code Review by Qodo</h3>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<h3>Sorry, something went wrong</h3>
We weren't able to complete the code review on our side. Please try again manually by commenting <code>/agentic_review</code> on this PR.
<br/>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>
<h3>PR Summary by Qodo</h3>
Parallelize span flushing across configurable worker processes
<code>✨ Enhancement</code> <code>🧪 Tests</code> <code>📝 Documentation</code> <code>⚙️ Configuration changes</code> <code>🕐 40+ Minutes</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<details>
<summary>AI Description</summary>
<dl>
<dd>
<br/>
><pre>
>• Parallelizes span flushing across shard groups with a configurable worker-process limit.
>• Isolates worker health, backpressure, memory accounting, restarts, and shutdown handling.
>• Adds concurrency coverage and guidance for safely narrowing union types.
></pre>
</dd>
</dl>
</details>
<details>
<summary>Diagram</summary>
<dl>
<dd>
<br/>
```mermaid
graph TD
CLI["Consumer CLI"] -->|configures| Factory["Strategy Factory"] -->|builds| Manager["Flusher Manager"] -->|assigns shards| Workers["Flusher Workers"]
Redis[("Span Redis")] -->|expired segments| Workers -->|produces| Kafka["Buffered Kafka"]
Manager -->|tracks| Health["Health Monitor"] -->|restarts| Workers
```
</dd>
</dl>
</details>
<details>
<summary>High-Level Assessment</summary>
<dl>
<dd>
<br/>
>The following are alternative approaches to this PR:
<details>
<summary><b>1. Dedicated process per shard</b></summary>
<dl>
<dd>
- ➕ Maximizes shard isolation and independent flushing throughput.
- ➕ Simplifies mapping between health signals and individual shards.
- ➖ Process count grows directly with partition count.
- ➖ Increases memory, Kafka producer connections, and operating overhead.
</dd>
</dl>
</details>
<details>
<summary><b>2. Shared thread pool</b></summary>
<dl>
<dd>
- ➕ Uses fewer resources and avoids process startup costs.
- ➕ Allows dynamic task scheduling across shards.
- ➖ Provides weaker failure isolation than subprocesses.
- ➖ Requires more synchronization around continuous shard polling and shared clients.
</dd>
</dl>
</details>
>**Recommendation:** The bounded subprocess approach is preferable because it preserves process isolation while preventing partition counts from forcing unbounded resource usage. Round-robin shard grouping also fits the existing shard-oriented buffer model; the CLI should continue defaulting conservatively while operators tune concurrency for workload and capacity.
</dd>
</dl>
</details>
<details>
<summary> Files changed (6) <code> +199 / -50 </code> </summary>
<dl>
<dd>
<br/>
<details>
<summary>Enhancement (2) <code> +130 / -47 </code></summary>
<dl>
<dd>
<details>
<summary>factory.py<code>Forward flusher concurrency into the processing strategy</code> <code>+3/-0</code></summary>
<br/>
>Forward flusher concurrency into the processing strategy
>
><pre>
>• Accepts the configured flusher-process limit and passes it to 'SpanFlusher' when constructing the partition-specific processing pipeline.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-973384a59a8a97829c8a4ceb385fc4fa74a3ac9b250191da9e53ef22f95a518e'>src/sentry/spans/consumers/process/factory.py</a>
</details>
<details>
<summary>flusher.py<code>Distribute span shards across managed flusher workers</code> <code>+127/-47</code></summary>
<br/>
>Distribute span shards across managed flusher workers
>
><pre>
>• Replaces the single flusher worker with bounded, round-robin shard groups and dedicated buffers. Tracks health, backpressure, restart counts, memory usage, metrics, and shutdown state per worker while retaining shared drift and stop signals.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-8a72390e3aabc90187bdebd196e90768eea631c168fc30d93bfa72a79f0b5077'>src/sentry/spans/consumers/process/flusher.py</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Tests (2) <code> +49 / -2 </code></summary>
<dl>
<dd>
<details>
<summary>test_consumer.py<code>Cover bounded flusher process distribution</code> <code>+48/-1</code></summary>
<br/>
>Cover bounded flusher process distribution
>
><pre>
>• Makes consumer tests transaction-aware, allows worker threads time to react to drift changes, and verifies four shards are distributed across a two-worker limit.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-d189bbca1faeaa4961e418aab1e268a5adc450b4bbdb7be80f0031ea5996d94d'>tests/sentry/spans/consumers/process/test_consumer.py</a>
</details>
<details>
<summary>test_flusher.py<code>Update backpressure assertion for multiple workers</code> <code>+1/-1</code></summary>
<br/>
>Update backpressure assertion for multiple workers
>
><pre>
>• Adapts the flusher backpressure test to validate the new per-worker backpressure state collection.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-3b147d384ac23bbfeb04bdaf6da914d521d8fbfe5e4b1b43e18ee0ecd198a0da'>tests/sentry/spans/consumers/process/test_flusher.py</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Documentation (1) <code> +11 / -0 </code></summary>
<dl>
<dd>
<details>
<summary>CLAUDE.md<code>Document safe union type narrowing</code> <code>+11/-0</code></summary>
<br/>
>Document safe union type narrowing
>
><pre>
>• Adds Python guidance discouraging 'hasattr' checks for union narrowing and recommends 'isinstance' before invoking type-specific methods.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-6ebdb617a8104a7756d0cf36578ab01103dc9f07e4dc6feb751296b9c402faf7'>CLAUDE.md</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Other (1) <code> +9 / -1 </code></summary>
<dl>
<dd>
<details>
<summary>__init__.py<code>Add flusher process-count CLI option</code> <code>+9/-1</code></summary>
<br/>
>Add flusher process-count CLI option
>
><pre>
>• Extends the span-processing consumer options with '--flusher-processes'. The option defaults to one and is forwarded to the strategy factory.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/72/files#diff-e5ee943f84dc1d6870ab01db646066449461e8f35ebdfab60048a4b3b6f73c59'>src/sentry/consumers/__init__.py</a>
</details>
</dd>
</dl>
</details>
</dd>
</dl>
</details>