Treating min_checkpoint_frequency=0 as 'disabled' stalls the changefeed highwater
Tier 2 easy go · local · real merged PR from cockroachdb/cockroach #153758
Ground-truth defects
The refactor moved the interval guard into canSave as `if interval <= 0 { return false }`, but min_checkpoint_frequency=0 is a valid setting meaning 'save as often as possible.' `<= 0` folds the zero case into the disabled case, so the changefeed highwater never advances and resolved timestamps are never emitted. The fix is `<= 0` -> `< 0`.
The change under review
diff (truncated — full diff)
@@ -203,6 +203,7 @@ go_test( "alter_changefeed_test.go", "changefeed_dist_test.go", "changefeed_job_info_test.go", + "changefeed_processors_test.go", "changefeed_progress_test.go", "changefeed_stmt_test.go", "changefeed_test.go", @@ -1524,13 +1524,7 @@ func TestAlterChangefeedAddTargetsDuringBackfill(t *testing.T) { defer rndMu.Unlock() if r.Span.Equal(fooTableSpan) { - // Do not emit resolved events for the entire table span. - // We "simulate" large table by splitting single table span into many parts, so - // we want to resolve those sub-spans instead of the entire table span. - // However, we have to emit something -- otherwise the entire changefeed - // machine would not work. - r.Span.EndKey = fooTableSpan.Key.Next() - return false, nil + return true, nil } if haveGaps { return rndMu.rnd.Intn(10) > 7, nil @@ -106,9 +106,11 @@ type changeAggregator struct { // eventConsumer consumes the event. eventConsumer eventConsumer - nextHighWaterFlush time.Time // next time high watermark may be flushed. - flushFrequency time.Duration // how often high watermark can be checkpointed. - lastSpanFlush time.Time // last time expensive, span based checkpoint was written. + flushFrequency time.Duration // how often high watermark can be checkpointed. + + // frontierFlushLimiter is a rate limiter for flushing the span frontier + // to the coordinator. + frontierFlushLimiter *saveRateLimiter // frontier keeps track of resolved timestamps for spans along with schema change // boundary information. @@ -281,6 +283,22 @@ func newChangeAggregatorProcessor( ca.flushFrequency = changefeedbase.DefaultMinCheckpointFrequency } + ca.frontierFlushLimiter, err = newSaveRateLimiter(saveRateConfig{ + name: "frontier", + intervalName: func() redact.SafeValue { + return redact.SafeString(changefeedbase.OptMinCheckpointFrequency) + }, + interval: func() time.Duration { + return ca.flushFrequency + }, + jitter: func() float64 { + return aggregatorFlushJitter.Get(&ca.FlowCtx.Cfg.Settings.SV) + }, + }, timeutil.DefaultTimeSource{}) + if err != nil { + return nil, err + } + return ca, nil } @@ -461,9 +479,6 @@ func (ca *changeAggregator) Start(ctx context.Context) { // Init heartbeat timer. ca.lastPush = timeutil.Now() - - // Generate expensive checkpoint only after we ran for a while. - ca.lastSpanFlush = timeutil.Now() } func (ca *changeAggregator) startKVFeed( @@ -730,18 +745,6 @@ var aggregatorFlushJitter = settings.RegisterFloatSetting( settings.WithPublic, ) -func nextFlushWithJitter(s timeutil.TimeSource, d time.Duration, j float64) (time.Time, error) { - if j < 0 || d < 0 { - return s.Now(), errors.AssertionFailedf("invalid jitter value: %f, duration: %s", j, d) - } - maxJitter := int64(j * float64(d)) - if maxJitter == 0 { - return s.Now().Add(d), nil - } - nextFlush := d + time.Duration(rand.Int63n(maxJitter)) - return s.Now().Add(nextFlush), nil -} - // Next is part of the RowSource interface. func (ca *changeAggregator) Next() (rowenc.EncDatumRow, *execinfrapb.ProducerMetadata) { shouldEmitHeartBeat := func() bool { @@ -897,7 +900,7 @@ func (ca *changeAggregator) flushBufferedEvents(ctx context.Context) error { // noteResolvedSpan periodically flushes Frontier progress from the current // changeAggregator node to the changeFrontier node to allow the changeFrontier // to persist the overall changefeed's progress -func (ca *changeAggregator) noteResolvedSpan(resolved jobspb.ResolvedSpan) (returnErr error) { +func (ca *changeAggregator) noteResolvedSpan(resolved jobspb.ResolvedSpan) error { ctx, sp := tracing.ChildSpan(ca.Ctx(), "changefeed.aggregator.note_resolved_span") defer sp.Finish() @@ -935,32 +938,17 @@ func (ca *changeAggregator) noteResolvedSpan(resolved jobspb.ResolvedSpan) (retu // TODO(yevgeniy): Consider doing something similar to how job checkpointing // works in the frontier where if we missed the window to checkpoint, we will attempt // the checkpoint at the next opportune moment. - checkpointFrontier := advanced && - (forceFlush || timeutil.Now().After(ca.nextHighWaterFlush)) + checkpointFrontier := (advanced && forceFlush) || ca.frontierFlushLimiter.canSave(ctx) if checkpointFrontier { - defer func() { - ca.nextHighWaterFlush, err = nextFlushWithJitter( - timeutil.DefaultTimeSource{}, ca.flushFrequency, aggregatorFlushJitter.Get(sv)) - if err != nil { - returnErr = errors.CombineErrors(returnErr, err) - } - }() - return ca.flushFrontier(ctx) + now := timeutil.Now() + if err := ca.flushFrontier(ctx); err != nil { + return err + } + ca.frontierFlushLimiter.doneSave(timeutil.Since(now)) } - // At a lower frequency, we checkpoint specific spans in the job progress - // either in backfills or if the highwater mark is excessively lagging behind. - checkpointSpans := (ca.frontier.InBackfill(resolved) || ca.frontier.HasLaggingSpans(sv)) && - canCheckpointSpans(sv, ca.lastSpanFlush) - - if checkpointSpans { - defer func() { - ca.lastSpanFlush = timeutil.Now() - }() - return ca.flushFrontier(ctx) - } - return returnErr + return nil } // flushFrontier flushes sink and emits resolved spans to the change frontier. @@ -1313,8 +1301,18 @@ func newChangeFrontierProcessor( cf.freqEmitResolved = emitNoResolved } - cf.frontierPersistenceLimiter = newSaveRateLimiter( - "frontier" /* name */, changefeedbase.FrontierPersistenceInterval) + cf.frontierPersistenceLimiter, err = newSaveRateLimiter(saveRateConfig{ + name: "frontier", + intervalName: func() redact.SafeValue { + return changefeedbase.FrontierPersistenceInterval.Name() + }, + interval: func() time.Duration { + return changefeedbase.FrontierPersistenceInterval.Get(&cf.FlowCtx.Cfg.Settings.SV) + }, + }, timeutil.DefaultTimeSource{}) + if err != nil { + return nil, err + } encodingOpts, err := opts.GetEncodingOptions() if err != nil { @@ -1919,7 +1917,7 @@ func (cf *changeFrontier) maybePersistFrontier(ctx context.Context) error { if cf.spec.JobID == 0 || !cf.evalCtx.Settings.Version.IsActive(ctx, clusterversion.V25_4) || - !cf.frontierPersistenceLimiter.canSave(ctx, &cf.FlowCtx.Cfg.Settings.SV) { + !cf.frontierPersistenceLimiter.canSave(ctx) { return nil } @@ -2349,10 +2347,12 @@ func shouldCountUsageError(err error) bool { status.Code(err) != codes.Canceled } -// durationSetting is a duration cluster setting. -type durationSetting interface { - Name() settings.SettingName - Get(sv *settings.Values) time.Duration +// saveRateConfig is the config for a saveRateLimiter. +type saveRateConfig struct { + name redact.SafeString + intervalName func() redact.SafeValue + interval func() time.Duration + jitter func() float64 // optional } // saveRateLimiter is a rate limiter for saving a piece of progress. @@ -2360,30 +2360,49 @@ type durationSetting interface { // It also limits saving to not be more frequent than the average // duration it takes to save progress. type saveRateLimiter struct { - name redact.SafeString - saveInterval durationSetting - warnEveryN util.EveryN + config saveRateConfig + warnEveryN util.EveryN + + clock timeutil.TimeSource lastSave time.Time avgSaveDuration time.Duration } // newSaveRateLimiter returns a new saveRateLimiter. -func newSaveRateLimiter(name redact.SafeString, saveInterval durationSetting) *saveRateLimiter { - return &saveRateLimiter{ - name: name, - saveInterval: saveInterval, - warnEveryN: util.Every(time.Minute), +func newSaveRateLimiter( + config saveRateConfig, clock timeutil.TimeSource, +) (*saveRateLimiter, error) { + if len(config.name) == 0 { + return nil, errors.AssertionFailedf("name is required") + } + if config.intervalName == nil { + return nil, errors.AssertionFailedf("interval name is required") + } + if config.interval == nil { + return nil, errors.AssertionFailedf("interval is required") } + return &saveRateLimiter{ + config: config, + warnEveryN: util.Every(time.Minute), + clock: clock, + }, nil } // canSave returns whether enough time has passed to save progress again. -func (l *saveRateLimiter) canSave(ctx context.Context, sv *settings.Values) bool { - interval := l.saveInterval.Get(sv) - if interval == 0 { +func (l *saveRateLimiter) canSave(ctx context.Context) bool { + interval := l.config.interval() + if interval <= 0 { return false } - now := timeutil.Now() + if l.config.jitter != nil { + if jitter := l.config.jitter(); jitter > 0 { + if maxJitter := time.Duration(jitter * float64(interval)); maxJitter > 0 { + interval += time.Duration(rand.Int63n(int64(maxJitter) + 1)) + } + } + } + now := l.clock.Now() elapsed := now.Sub(l.lastSave) if elapsed < interval { return false @@ -2393,8 +2412,7 @@ func (l *saveRateLimiter) canSave(ctx context.Context, sv *settings.Values) bool log.Changefeed.Warningf(ctx, "cannot save %s even though %s has elapsed "+ "since last save and %s is set to %s because average duration to save was %s "+ "and further saving is disabled until that much time elapses", - l.name, elapsed, l.saveInterval.Name(), - interval, l.avgSaveDuration) + l.config.name, elapsed, l.config.intervalName(), interval, l.avgSaveDuration) } return false } @@ -2404,7 +2422,7 @@ func (l *saveRateLimiter) canSave(ctx context.Context, sv *settings.Values) bool // doneSave must be called after each save is completed with the duration // it took to save progress. func (l *saveRateLimiter) doneSave(saveDuration time.Duration) { - l.lastSave = timeutil.Now() + l.lastSave = l.clock.Now() // Update the average save duration using an exponential moving average. if l.avgSaveDuration == 0 { new file mode 100644 @@ -0,0 +1,154 @@ +// Copyright 2025 The Cockroach Authors. +// +// Use of this software is governed by the CockroachDB Software License +// included in the /LICENSE file. + +package changefeedccl + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/cockroachdb/cockroach/pkg/util/leaktest" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/timeutil" + "github.com/cockroachdb/redact" + "github.com/stretchr/testify/require" +) + +func TestSaveRateLimiter(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + ctx := context.Background() + + intervals := map[string]time.Duration{ + "positive interval": 30 * time.Second, + "negative interval": -30 * time.Second, + "zero interval": 0, + "very small interval": time.Nanosecond, + } + + jitters := map[string]float64{ + "positive jitter": 0.1, + "negative jitter": -0.1, + "zero jitter": 0, + "very small jitter": 1e-12, + } + + for intervalName, interval := range intervals { + for jitterName, jitter := range jitters { + t.Run(fmt.Sprintf("%s with %s", intervalName, jitterName), func(t *testing.T) { + // Set up the mock clock for testing. + now := timeutil.Now() + clock := timeutil.NewManualTime(now) + + // Create the save rate limiter. + l, err := newSaveRateLimiter(saveRateConfig{ + name: "test", + intervalName: func() redact.SafeValue { + return redact.SafeString(intervalName) + }, + interval: func() time.Duration { + return interval + }, + jitter: func() float64 { + return jitter + }, + }, clock) + require.NoError(t, err) + + // A non-positive interval indicates that saving is disabled so we only + // need to test that we can't save at all. + if interval <= 0 { + require.False(t, l.canSave(ctx)) + clock.Advance(24 * time.Hour) + require.False(t, l.canSave(ctx)) + return + } + + // We can do one save right away if the interval. + require.True(t, l.canSave(ctx)) + l.doneSave(0 /* saveDuration */) + + // Can't immediately save again. + require.False(t, l.canSave(ctx)) + + // Make sure interval and jitter works correctly. + var maxJitter time.Duration + if jitter > 0 { + maxJitter = time.Duration(jitter * float64(interval)) + } + clock.Advance(interval + maxJitter) + require.True(t, l.canSave(ctx)) + + // Set the save duration to something high to make sure we can't save + // due to high average save duration. + l.doneSave(time.Hour) + clock.Advance(interval + maxJitter) + require.False(t, l.canSave(ctx)) + clock.Advance(time.Hour - (interval + maxJitter)) + require.True(t, l.canSave(ctx)) + + // Set the save duration to something even higher to make sure the + // average algorithm works. + l.doneSave(2 * time.Hour) + clock.Advance(time.Hour) + require.False(t, l.canSave(ctx)) + clock.Advance(time.Hour) + require.True(t, l.canSave(ctx)) + l.doneSave(0 /* saveDuration */) + }) + } + } +} + +func TestSaveRateLimiterError(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + name := redact.SafeString("test") + intervalName := func() redact.SafeValue { + return redact.SafeString("interval") + } + interval := func() time.Duration { + return 30 * time.Second + } + + for testName, tc := range map[string]struct { + config saveRateConfig + expectedErr string + }{ + "missing name": { + config: saveRateConfig{ + intervalName: intervalName, + interval: interval, + }, + expectedErr: "name is required", + }, + "missing interval name": { + config: saveRateConfig{ + name: name, + interval: interval, + }, + expectedErr: "interval name is required", + }, + "missing interval": { + config: saveRateConfig{ + name: name, + intervalName: intervalName, + }, + expectedErr: "interval is required", + }, + } { + t.Run(testName, func(t *testing.T) { + if tc.expectedErr == "" { + t.Fatal("missing expected error") + } + _, err := newSaveRateLimiter(tc.config, timeutil.DefaultTimeSource{}) + require.ErrorContains(t, err, tc.expectedErr) + }) + } +} @@ -9299,7 +9299,7 @@ func TestCoreChangefeedBackfillScanCheckpoint(t *testing.T) { return nil } - foo := feed(t, f, `CREATE CHANGEFEED FOR TABLE foo`) + foo := feed(t, f, `CREATE CHANGEFEED FOR TABLE foo WITH min_checkpoint_frequency='1ns'`) defer closeFeed(t, foo) payloads := make([]string, rowCount+1) @@ -9364,139 +9364,6 @@ func TestCheckpointFrequency(t *testing.T) { require.False(t, js.progressUpdatesSkipped) } -func TestFlushJitter(t *testing.T) { - defer leaktest.AfterTest(t)() - defer log.Scope(t).Close(t) - - // Test the logic around applying jitter to the flush logic. - // The more involved test that would try to capture flush times would likely - // be pretty flaky due to the fact that flush times do not happen at exactly - // min_flush_frequency period, and thus it would be hard to tell if the - // difference is due to jitter or not. Just verify nextFlushWithJitter function - // works as expected with controlled time source. - - ts := timeutil.NewManualTime(timeutil.Now()) - const numIters = 100 - - for _, tc := range []struct { - flushFrequency time.Duration - jitter float64 - expectedFlushDuration time.Duration - expectedErr bool - }{ - // Negative jitter. - { - flushFrequency: -1, - jitter: -0.1, - expectedFlushDuration: 0, - expectedErr: true, - }, - { - flushFrequency: 0, - jitter: -0.1, - expectedFlushDuration: 0, - expectedErr: true, - }, - { - flushFrequency: 10 * time.Millisecond, - jitter: -0.1, - expectedFlushDuration: 0, - expectedErr: true, - }, - { - flushFrequency: 100 * time.Millisecond, - jitter: -0.1, - expectedFlushDuration: 0, - expectedErr: true,
Leaderboard for this case
easy · 8 of 9 bots fully caught it (89% solve rate) · 89% avg coverage.
| # | Bot | Result | Caught | Coverage | Findings | False alarms |
|---|---|---|---|---|---|---|
| 1 | Claude Code (Opus 5.5) | solved | 1/1 | 100.0% | 3 | 0 |
| 2 | Cubic | solved | 1/1 | 100.0% | 1 | 0 |
| 3 | cursor | solved | 1/1 | 100.0% | 2 | 0 |
| 4 | Drape (Deep Review) | solved | 1/1 | 100.0% | 2 | 0 |
| 5 | Drape (Review) | solved | 1/1 | 100.0% | 4 | 0 |
| 6 | greptile | solved | 1/1 | 100.0% | 3 | 0 |
| 7 | Mjolnir (Opus 5.5) | solved | 1/1 | 100.0% | 4 | 0 |
| 8 | Mjolnir (Sonnet 5.5) | solved | 1/1 | 100.0% | 3 | 0 |
| 9 | qodo | miss | 0/1 | 0.0% | 2 | 0 |
Bot results
Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
Jitter is now redrawn inside canSave on every call. noteResolvedSpan calls canSave for each resolved span, so once elapsed passes the base interval, any call that draws a small jitter succeeds. Flushes therefore cluster near the base interval. The removed nextFlushWithJitter fixed one jittered deadline per flush. [grounded: diff] [consensus 3/3: valid=3]
The old code flushed only when the frontier advanced, or on the span-checkpoint path, which was throttled by span_checkpoint.interval. The new code flushes whenever canSave allows it, whether or not the frontier advanced, and the span-checkpoint throttle is gone. Flush frequency really does go up for non-advancing frontiers. The change looks deliberate, and the extra cost is probably small. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
Changefeeds created with `min_checkpoint_frequency='0s'` (a valid value) will stop reporting progress. Before this change, `timeutil.Now().After(ca.nextHighWaterFlush)` was always true when the frequency was 0, so the aggregator flushed resolved spans on every advance. The aggregator now uses `saveRateLimiter.canSave`, which returns false whenever the interval is <= 0. With 0s, the aggregator only flushes on schema-change boundary spans (`forceFlush`). Under normal operation the change frontier gets no resolved spans, so the highwater mark, resolved messages and job checkpoints stop advancing, and the protected timestamp is not moved forward. The test in `show_changefeed_jobs_test.go` was changed from `min_checkpoint_frequency='0s'` to `'1s'`, which hides this regression.
Jitter is now drawn again on every `canSave` call instead of once per flush. Because `noteResolvedSpan` calls `canSave` many times per second, the first call after `elapsed >= interval` that happens to draw a small jitter will succeed. In practice flushes land close to the plain interval, so `changefeed.aggregator.flush_jitter` mostly stops spreading out aggregator flushes. Previously `nextFlushWithJitter` picked one deadline per flush cycle. The jittered deadline should be computed once in `doneSave` and stored.
The aggregator now flushes, including a sink flush and a resolved-span emit, every `min_checkpoint_frequency` even when the frontier has not advanced. The old code only did this when the frontier advanced, or during a backfill or with lagging spans, and in those cases it was throttled by `changefeed.span_checkpoint.interval`. The separate span-checkpoint throttle (`canCheckpointSpans`/`lastSpanFlush`) has been removed. As a result, idle or steady-state changefeeds do more sink flushes and send more frontier traffic to the coordinator than before.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
raw comments (2)
<!-- cubic:v=8c4ec30e-d54d-48f7-afe2-627ad948cf27 -->
<!-- cubic:review-run=57670b81-731f-5849-8244-aff1d44b73b6 -->
<!-- metadata:{"confidence":9} -->
P2: When `SpanCheckpointInterval` is shorter than `min_checkpoint_frequency`, lagging or backfill span progress is no longer checkpointed at the configured span-checkpoint cadence. This can delay span checkpoints by the aggregator interval, or suppress them when `min_checkpoint_frequency=0`, increasing duplicate reprocessing after a restart. Preserve an independent span-checkpoint flush trigger for backfills and lagging spans.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At pkg/ccl/changefeedccl/changefeed_processors.go, line 941:
<comment>When `SpanCheckpointInterval` is shorter than `min_checkpoint_frequency`, lagging or backfill span progress is no longer checkpointed at the configured span-checkpoint cadence. This can delay span checkpoints by the aggregator interval, or suppress them when `min_checkpoint_frequency=0`, increasing duplicate reprocessing after a restart. Preserve an independent span-checkpoint flush trigger for backfills and lagging spans.</comment>
<file context>
@@ -935,32 +938,17 @@ func (ca *changeAggregator) noteResolvedSpan(resolved jobspb.ResolvedSpan) (retu
// the checkpoint at the next opportune moment.
- checkpointFrontier := advanced &&
- (forceFlush || timeutil.Now().After(ca.nextHighWaterFlush))
+ checkpointFrontier := (advanced && forceFlush) || ca.frontierFlushLimiter.canSave(ctx)
if checkpointFrontier {
</file context>
```
</details>
<!-- cubic:review-summary:start -->
**1 issue found** across 7 files
<!-- cubic:review-summary:end -->
<details>
<summary>Prompt for AI agents (unresolved issues)</summary>
```text
Check if these issues are valid — if so, understand the root cause of each and fix them. If appropriate, use sub-agents to investigate and fix each issue separately.
<file name="pkg/ccl/changefeedccl/changefeed_processors.go">
<violation number="1" location="pkg/ccl/changefeedccl/changefeed_processors.go:941">
P2: When `SpanCheckpointInterval` is shorter than `min_checkpoint_frequency`, lagging or backfill span progress is no longer checkpointed at the configured span-checkpoint cadence. This can delay span checkpoints by the aggregator interval, or suppress them when `min_checkpoint_frequency=0`, increasing duplicate reprocessing after a restart. Preserve an independent span-checkpoint flush trigger for backfills and lagging spans.</violation>
</file>
```
</details>
<sub>Reply with feedback, questions, or to request a fix.<br /><br />[Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/27/ai_pr_review_github_delivery_6abed358688d379dd92e13b9811b92450b64088144c12b16ebaf440548aad327?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F27)</sub>
<!-- cubic:review-post:ai_pr_review_github_delivery_6abed358688d379dd92e13b9811b92450b64088144c12b16ebaf440548aad327:39d1008bae00bb01bfd9e0fb9255985f2f5ce3da:57670b81-731f-5849-8244-aff1d44b73b6 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
The old code fixed `nextHighWaterFlush` once after each flush. The new `canSave` draws a fresh random jitter on every call. With frequent resolved-span calls, a small draw soon follows once the base interval has passed, so flushes land near the base interval and flush_jitter mostly stops spreading them out. [grounded: diff] [consensus 5/5: valid=5]
raw comments (3)
### Zero interval disables frontier flushes
**High Severity**
<!-- DESCRIPTION START -->
`canSave` treats a non-positive `interval` as disabled, so `min_checkpoint_frequency='0s'` now suppresses periodic aggregator flushes. Previously `0` meant no minimum delay and every advancing resolved span was flushed. High-water, resolved timestamps, and span checkpoints now stall except at schema boundaries.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: 3918ec00-fc7c-4bd7-b106-d76b854837db -->
<!-- LOCATIONS START
pkg/ccl/changefeedccl/changefeed_processors.go#L2393-L2396
pkg/ccl/changefeedccl/changefeed_processors.go#L940-L941
LOCATIONS END -->
<details>
<summary>Additional Locations (1)</summary>
- [`pkg/ccl/changefeedccl/changefeed_processors.go#L940-L941`](https://github.com/drape-io/crbench-cursor-2026-09/blob/ecd4f075bd9bd34039599fb6160b8dfe9f9d51fe/pkg/ccl/changefeedccl/changefeed_processors.go#L940-L941)
</details>
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmE0NTFmOGMzLTg4ODEtNDRlZi1iNmExLWU1Yjg1MGIwYThiOCIsImVuY3J5cHRpb25LZXkiOiJOUVFvc1BWWEJ2b0lOc0UzSVJTQnNNUGtQX1FqY0VYd1ZBa3VVSGwwdWNrIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE1Mzc1OCIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmE0NTFmOGMzLTg4ODEtNDRlZi1iNmExLWU1Yjg1MGIwYThiOCIsImVuY3J5cHRpb25LZXkiOiJOUVFvc1BWWEJ2b0lOc0UzSVJTQnNNUGtQX1FqY0VYd1ZBa3VVSGwwdWNrIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE1Mzc1OCIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjkxLCJjb21taXRTaGEiOiJlY2Q0ZjA3NWJkOWJkMzQwMzk1OTlmYjYxNjBiOGRmZTlmOWQ1MWZlIiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit ecd4f075bd9bd34039599fb6160b8dfe9f9d51fe. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
### Jitter rerolled on every check
**Medium Severity**
<!-- DESCRIPTION START -->
`canSave` draws a new jittered interval on every call instead of fixing one deadline after each save. Frequent resolved-span checks then succeed as soon as the base interval elapses, so `changefeed.aggregator.flush_jitter` no longer spreads aggregator flushes and they tend to hit the coordinator together.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: 69feaee7-2fdf-4f78-b361-e0a0801cf25a -->
<!-- LOCATIONS START
pkg/ccl/changefeedccl/changefeed_processors.go#L2397-L2403
LOCATIONS END -->
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmNhODZhMjE2LTY1NWYtNDgxNS1hZjcxLTdhOTIxY2ZiOGViMyIsImVuY3J5cHRpb25LZXkiOiJyM0xISXBJS2ZuUmdQYkhDN2lmTDlsRjF0cl9GMlR6R09BNm9XQzNRZWpZIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE1Mzc1OCIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmNhODZhMjE2LTY1NWYtNDgxNS1hZjcxLTdhOTIxY2ZiOGViMyIsImVuY3J5cHRpb25LZXkiOiJyM0xISXBJS2ZuUmdQYkhDN2lmTDlsRjF0cl9GMlR6R09BNm9XQzNRZWpZIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE1Mzc1OCIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjkxLCJjb21taXRTaGEiOiJlY2Q0ZjA3NWJkOWJkMzQwMzk1OTlmYjYxNjBiOGRmZTlmOWQ1MWZlIiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit ecd4f075bd9bd34039599fb6160b8dfe9f9d51fe. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_REVIEW -->
Cursor Bugbot has reviewed your changes using high effort and found 2 potential issues.
<!-- BUGBOT_FIX_ALL -->
<a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aTplMmYwM2M4ZC02YTU3LTRmNWMtYWJiYi1iNTk0NmI1NDU5YTAiLCJlbmNyeXB0aW9uS2V5IjoiaE1lRVpyNHNodXRsQThnUG5VWEtrRUNneGdlQVNWZzFuWkxaZUZDd1huUSIsImJyYW5jaCI6InQyLWNvY2tyb2FjaC0xNTM3NTgiLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>
<!-- /BUGBOT_FIX_ALL -->
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN -->
<sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END -->
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit ecd4f075bd9bd34039599fb6160b8dfe9f9d51fe. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
The new canSave draws a fresh random jitter on every call, and noteResolvedSpan calls it once per resolved span. Once the base interval has passed, frequent calls make it very likely that some draw is small, so the flush happens near the base interval and most of the jitter is lost. The removed nextFlushWithJitter picked the jittered deadline once per flush. This regression comes from this change. [grounded: tree pkg/ccl/changefeedccl/changefeed_processors.go] [consensus 3/3: valid=3]
raw comments (2)
**min_checkpoint_frequency='0s' stops aggregator progress flushes**
`canSave` treats a zero interval as disabled. Before, 0 meant flush on every advance. Now these feeds flush only at schema-change boundaries, so highwater stalls. For the aggregator limiter, treat a zero interval as "always save".
```
pkg/ccl/changefeedccl/changefeed_processors.go:286 ca.frontierFlushLimiter, err = newSaveRateLimiter(saveRateConfig{
pkg/ccl/changefeedccl/changefeed_processors.go:287 name: "frontier",
pkg/ccl/changefeedccl/changefeed_processors.go:288 intervalName: func() redact.SafeValue {
pkg/ccl/changefeedccl/changefeed_processors.go:289 return redact.SafeString(changefeedbase.OptMinCheckpointFrequency)
pkg/ccl/changefeedccl/changefeed_processors.go:938 (base) checkpointFrontier := advanced &&
pkg/ccl/changefeedccl/changefeed_processors.go:939 (base) (forceFlush || timeutil.Now().After(ca.nextHighWaterFlush))
pkg/ccl/changefeedccl/changefeed_processors.go:941 (base) if checkpointFrontier {
pkg/ccl/changefeedccl/changefeed_processors.go:942 (base) defer func() {
pkg/ccl/changefeedccl/changefeed_processors.go:943 (base) ca.nextHighWaterFlush, err = nextFlushWithJitter(
pkg/ccl/changefeedccl/changefeed_processors.go:944 (base) timeutil.DefaultTimeSource{}, ca.flushFrequency, aggregatorFlushJitter.Get(sv))
```
**Flush jitter re-rolled every call, so mostly lost**
`canSave` draws new jitter on every call, once per resolved span. After the base interval, some draw quickly lands near zero, so aggregators flush almost in step. Pick the jittered interval once in `doneSave` and compare against it in `canSave`.
```
pkg/ccl/changefeedccl/changefeed_processors.go:2398 if l.config.jitter != nil {
pkg/ccl/changefeedccl/changefeed_processors.go:2399 if jitter := l.config.jitter(); jitter > 0 {
pkg/ccl/changefeedccl/changefeed_processors.go:2400 if maxJitter := time.Duration(jitter * float64(interval)); maxJitter > 0 {
pkg/ccl/changefeedccl/changefeed_processors.go:2401 interval += time.Duration(rand.Int63n(int64(maxJitter) + 1))
pkg/ccl/changefeedccl/changefeed_processors.go:942 (base) defer func() {
pkg/ccl/changefeedccl/changefeed_processors.go:943 (base) ca.nextHighWaterFlush, err = nextFlushWithJitter(
pkg/ccl/changefeedccl/changefeed_processors.go:944 (base) timeutil.DefaultTimeSource{}, ca.flushFrequency, aggregatorFlushJitter.Get(sv))
pkg/ccl/changefeedccl/changefeed_processors.go:945 (base) if err != nil {
pkg/ccl/changefeedccl/changefeed_processors.go:946 (base) returnErr = errors.CombineErrors(returnErr, err)
pkg/ccl/changefeedccl/changefeed_processors.go:947 (base) }
```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
Jitter is now re-sampled on every canSave call. The aggregator calls canSave for each resolved span, so once elapsed passes the base interval, a small draw soon passes the check. The old code fixed the jitter once per flush in nextFlushWithJitter. Most of the jitter's spreading effect is lost. [grounded: tree pkg/ccl/changefeedccl/changefeed_processors.go] [consensus 3/3: valid=3]
The time-based path no longer requires `advanced`. This looks deliberate: it replaces the removed span-checkpoint path, which also flushed without the frontier advancing during backfills. Flushing at the rate-limited interval is the intended purpose of min_checkpoint_frequency, so calling it a defect is doubtful. [grounded: diff] [consensus 3/5: valid=3, invalid=1, uncertain=1]
The flushFrontier duration, which includes sink.Flush, feeds avgSaveDuration. canSave then blocks saving until that average has elapsed, so slow sinks get flushed less often than before. This matches the limiter's documented design, and min_checkpoint_frequency sets a minimum gap rather than a required rate. The impact is low. [grounded: diff] [consensus 3/5: valid=3, invalid=1, uncertain=1]
raw comments (4)
**Flush jitter is re-rolled on every check**
`canSave` draws new random jitter on every call, and the aggregator calls it for each resolved span. The check passes as soon as one draw is small, so the effective interval collapses to about the base interval. Pick jitter once per save in `doneSave` and store the next allowed time.
```
pkg/ccl/changefeedccl/changefeed_processors.go:2403 }
pkg/ccl/changefeedccl/changefeed_processors.go:2404 }
pkg/ccl/changefeedccl/changefeed_processors.go:2405 now := l.clock.Now()
pkg/ccl/changefeedccl/changefeed_processors.go:2406 elapsed := now.Sub(l.lastSave)
```
**Frontier flushed even when frontier did not advance**
`checkpointFrontier` is now `(advanced && forceFlush) || canSave(ctx)`, so the time-based path no longer requires `advanced`. The old code required it. Once the interval elapses, any non-advancing resolved span triggers a sink flush and an emit.
```
pkg/ccl/changefeedccl/changefeed_processors.go:941 checkpointFrontier := (advanced && forceFlush) || ca.frontierFlushLimiter.canSave(ctx)
pkg/ccl/changefeedccl/changefeed_processors.go:943 if checkpointFrontier {
pkg/ccl/changefeedccl/changefeed_processors.go:944 now := timeutil.Now()
pkg/ccl/changefeedccl/changefeed_processors.go:945 if err := ca.flushFrontier(ctx); err != nil {
```
**min_checkpoint_frequency=0 stops periodic flushes**
`canSave` returns false for any interval <= 0, and it is now the only periodic flush trigger. With `min_checkpoint_frequency='0s'` the aggregator used to flush on every advance; now it flushes only at boundaries. Treat a zero interval as always allowed for the aggregator limiter.
```
pkg/ccl/changefeedccl/changefeed_processors.go:941 checkpointFrontier := (advanced && forceFlush) || ca.frontierFlushLimiter.canSave(ctx)
pkg/ccl/changefeedccl/changefeed_processors.go:2394 interval := l.config.interval()
pkg/ccl/changefeedccl/changefeed_processors.go:2395 if interval <= 0 {
pkg/ccl/changefeedccl/changefeed_processors.go:2396 return false
pkg/ccl/changefeedccl/changefeed_processors.go:741 "jitter aggregator flushes as a fraction of min_checkpoint_frequency. This "+
pkg/ccl/changefeedccl/changefeed_processors.go:742 "setting has no effect if min_checkpoint_frequency is set to 0.",
```
**Slow sink flush throttles the frontier further**
The aggregator passes the `flushFrontier` time, which includes `sink.Flush`, to `doneSave`. `canSave` then refuses to save until the average duration has elapsed, so a slow sink is flushed less often than min_checkpoint_frequency says. Don't feed sink-flush time into the aggregator limiter's average.
```
pkg/ccl/changefeedccl/changefeed_processors.go:944 now := timeutil.Now()
pkg/ccl/changefeedccl/changefeed_processors.go:945 if err := ca.flushFrontier(ctx); err != nil {
pkg/ccl/changefeedccl/changefeed_processors.go:946 return err
pkg/ccl/changefeedccl/changefeed_processors.go:947 }
```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
canSave draws a new random jitter on every call. Resolved events call it often, so the first draw near the base interval lets the flush through, and the effective delay shrinks to about the minimum. The old code computed the jittered deadline once per flush (nextFlushWithJitter after each save). [grounded: diff] [consensus 3/3: valid=3]
The separate path that flushed the aggregator at changefeed.span_checkpoint.interval during backfill or lag was removed. Non-boundary flushes are now gated only by min_checkpoint_frequency (default 30s), so span checkpoints cannot happen more often than that. The tests had to be changed to set min_checkpoint_frequency='1ns' or '1s', which confirms the cadence changed. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Zero Frequency Stalls Progress**
A supported `min_checkpoint_frequency = 0` now disables every ordinary aggregator frontier flush. Non-boundary resolved events still advance the local frontier, but `canSave` always returns false and no other path forwards those updates to the coordinator. As a result, the changefeed's high-water progress can remain stalled indefinitely. Previously, zero made the next flush immediately eligible.
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Jitter Resamples Every Check**
Jitter is randomly resampled on every `canSave` call instead of selecting one deadline after each save. Since every resolved event calls this method, feeds with frequent resolved events get repeated chances to pass immediately after the base interval. This biases flushes toward the minimum interval and defeats the jitter intended to spread concurrent aggregator flushes. Store one jittered deadline per save cycle and test that repeated checks do not change it.
Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!
<h3>Greptile Summary</h3>
This PR generalizes the changefeed progress rate limiter, applies it to aggregator frontier flushes, and adds direct limiter and rolling-restart coverage. It also adjusts several checkpoint-related tests.
- Ordinary frontier emission is now controlled by `saveRateLimiter`.
- Coordinator frontier persistence adopts the generalized limiter configuration.
- Backfill and rolling-restart tests receive more aggressive checkpoint and range-splitting configurations.
- The new limiter introduces regressions for zero-frequency feeds and independent span-checkpoint scheduling, while its jitter is resampled too frequently.
<h3>Confidence Score: 3/5</h3>
The PR is not safe to merge because supported zero-frequency feeds can stop forwarding ordinary progress and span checkpoints no longer honor their independent configured cadence.
The generalized limiter disables non-boundary frontier emission at a valid zero interval and removes the independent aggregator flush opportunity required for timely span checkpoints; jitter resampling also weakens flush desynchronization.
**Files Needing Attention:** pkg/ccl/changefeedccl/changefeed_processors.go, pkg/ccl/changefeedccl/changefeed_processors_test.go
<details><summary><h3>Important Files Changed</h3></summary>
| Filename | Overview |
|----------|----------|
| pkg/ccl/changefeedccl/changefeed_processors.go | Generalizes progress rate limiting and changes aggregator flush scheduling, introducing zero-interval, span-checkpoint cadence, and jitter regressions. |
| pkg/ccl/changefeedccl/changefeed_processors_test.go | Adds direct limiter validation, but codifies problematic zero-interval behavior and does not test stable per-cycle jitter. |
| pkg/cmd/roachtest/tests/cdc.go | Adds frontier-persistence rolling-restart coverage and increases range distribution for checkpoint scenarios. |
| pkg/ccl/changefeedccl/changefeed_test.go | Updates core backfill checkpoint coverage to request frequent aggregator flushes. |
| pkg/ccl/changefeedccl/alter_changefeed_test.go | Simplifies filtering of whole-table resolved spans in the backfill target-alteration test. |
</details>
<details open><summary><h3>Flowchart</h3></summary>
```mermaid
%%{init: {'theme': 'neutral'}}%%
flowchart LR
R[KV feed resolved event] --> A[Aggregator local frontier]
A --> B{Boundary event?}
B -->|Yes| F[Flush sink and emit frontier]
B -->|No| L{saveRateLimiter allows save?}
L -->|Yes| F
L -->|No| W[Progress remains local]
F --> C[Coordinator frontier]
C --> S{Span checkpoint due?}
S -->|Yes| P[Persist job checkpoint]
S -->|No| H[Advance high water]
```
</details>
<!-- greptile_failed_comments -->
<details><summary><h3>Comments Outside Diff (1)</h3></summary>
1. `pkg/ccl/changefeedccl/changefeed_processors.go`, line 936-947 ([link](https://github.com/drape-io/crbench-greptile-2026-09/blob/2b4eb0d5d838d27aeddc3044659042b367f9d9f4/pkg/ccl/changefeedccl/changefeed_processors.go#L936-L947))
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Span Checkpoints Are Delayed**
Removing the independent span-checkpoint flush path makes `changefeed.span_checkpoint.interval` depend on `min_checkpoint_frequency`. During a backfill or when spans lag, a checkpoint due under the shorter span-checkpoint interval is not sent to the coordinator until the potentially longer aggregator interval elapses. A retry during that window therefore resumes from stale progress and repeats scan work despite the configured checkpoint cadence.
</details>
<!-- /greptile_failed_comments -->
<sub>Reviews (1): Last reviewed commit: ["Update pkg/ccl/changefeedccl/BUILD.bazel..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/2b4eb0d5d838d27aeddc3044659042b367f9d9f4) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=64288953)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | exact |
Before the change, a zero flushFrequency made the aggregator flush on every frontier advance. Now canSave returns false whenever interval <= 0, so with min_checkpoint_frequency=0 (which the options allow) the aggregator only flushes at schema-change boundaries. [grounded: diff] [consensus 3/3: valid=3]
canSave draws a new jitter on every call, and it runs on every resolved span. This pulls the effective flush threshold toward the base interval, unlike the old code, which drew jitter once per flush. [grounded: diff] [consensus 3/3: valid=3]
Same issue as finding 1: jitter is drawn again on every canSave call, which removes most of the spreading the old per-flush draw provided. [grounded: diff] [consensus 3/3: valid=3]
raw comments (4)
[P1] `min_checkpoint_frequency='0s'` now stops aggregator frontier flushes entirely
`min_checkpoint_frequency` is explicitly allowed to be zero (`durationOption.thatCanBeZero()` in changefeedbase/options.go:387). Before this change, 0 meant "flush on every frontier advance", because `nextHighWaterFlush` was always in the past. The aggregator limiter now passes `ca.flushFrequency` straight through as the interval, and `canSave` returns false for `interval ‹= 0`. So with 0 the aggregator only flushes resolved spans on schema-change boundaries (`advanced && forceFlush`). The changeFrontier then never receives progress: the highwater, resolved messages and PTS never advance, and the sink is never flushed through this path. Any existing or new changefeed created with `min_checkpoint_frequency='0s'` stalls silently. The edit to `show_changefeed_jobs_test.go`, which changes '0s' to '1s', works around this rather than fixing it. Zero needs to mean "always allowed" for the aggregator limiter, as it did before. The "0 disables saving" meaning should only apply to the frontier-persistence setting.
[P2] Jitter is re-sampled on every `canSave` call, so it barely spreads flushes
The old `nextFlushWithJitter` drew a random deadline once per flush. Now `canSave` adds a new `rand.Int63n(maxJitter+1)` on each call, and the aggregator calls it for every resolved span (many per second with lots of ranges). Each call after `interval` has elapsed is another independent chance to pass. So the flush fires almost as soon as `elapsed` exceeds `interval` plus a small fraction of `maxJitter`, not at a uniformly random point in `[interval, interval+maxJitter]`. In effect, `changefeed.aggregator.flush_jitter` mostly stops staggering flushes across aggregators, which defeats its purpose of avoiding synchronized flushes and checkpoint storms. Picking the jittered interval once in `doneSave` (or on the first check after a save) would keep the old behavior.
[P1] `min_checkpoint_frequency='0s'` now stops aggregator frontier flushes entirely
Before this change, `flushFrequency == 0` meant `nextHighWaterFlush` was always in the past, so the aggregator flushed every time its frontier advanced. That matches the documented behavior and the `aggregatorFlushJitter` description, which says the setting "has no effect if min_checkpoint_frequency is set to 0". The option is also registered with `durationOption.thatCanBeZero()`. With this change the check goes through `frontierFlushLimiter.canSave`, which returns false whenever `interval ‹= 0`. So for any changefeed created with `min_checkpoint_frequency='0s'`, the aggregator only sends resolved spans to the coordinator at schema-change boundaries (`advanced && forceFlush`). The highwater stops advancing, no resolved messages are emitted, and job progress isn't checkpointed. The test changes from `'0s'` to `'1s'`/`'1ns'` (for example, in `TestShowChangefeedJobsShowsHighWaterTimestamp`) avoid this case rather than cover it. The aggregator should treat a zero interval as "flush on every advance", not as "never flush".
[P2] Jitter is drawn again on every `canSave` call, which cancels most of its effect
`canSave` picks a new random jitter in `[0, jitter*interval]` on each call, and the aggregator now calls it on every resolved span it gets (many per second on a busy feed). Once `interval` has passed, each call is a fresh chance to draw a small enough jitter, so a flush happens almost right at `interval`. The old `nextFlushWithJitter` drew jitter once per flush to pick a fixed next flush time. That spread flushes from different aggregators apart, which is what `changefeed.aggregator.flush_jitter` exists to do. If the jitter is drawn once, after each `doneSave`, the old behavior is preserved.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✓ | off by one |
Before the change, a flushFrequency of 0 made nextHighWaterFlush equal to now, so every advance flushed. Now canSave returns false whenever the interval is <= 0, so with min_checkpoint_frequency='0s' only boundary (forceFlush) advances flush. The show_changefeed_jobs test was switched from '0s' to '1s', which fits this regression. [grounded: diff] [consensus 3/3: valid=3]
The removed branch flushed the aggregator frontier on the span_checkpoint.interval cadence during backfills and when spans were lagging. Now flushes happen only on the min_checkpoint_frequency cadence, so span checkpoints during a backfill are written less often (the tests and roachtest add min_checkpoint_frequency to compensate). The behavior change is real, though it may have been intended. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
[P2] min_checkpoint_frequency='0s' now stops aggregator flushes
The new `saveRateLimiter.canSave` returns false whenever `interval ‹= 0`. The aggregator now uses it as `frontierFlushLimiter`, with `interval` set to `ca.flushFrequency`. `min_checkpoint_frequency` is declared `thatCanBeZero()` (options.go:387). The `aggregatorFlushJitter` doc says it "has no effect if min_checkpoint_frequency is set to 0", so zero is a supported value. Before this change, zero meant `nextHighWaterFlush = now`, so the aggregator flushed on every advance. Now a changefeed created with `min_checkpoint_frequency='0s'` flushes the frontier to the coordinator only for boundary-forced flushes (schema change or terminal boundary). Otherwise it never sends progress, so the high-water mark and resolved timestamps stall. `show_changefeed_jobs_test.go` was edited from `'0s'` to `'1s'`, which hides the regression instead of fixing it. The `interval == 0` guard is correct for `FrontierPersistenceInterval`, where 0 means disabled. It should not apply to the aggregator, where 0 means no throttling.
[P1] min_checkpoint_frequency='0s' no longer flushes the aggregator frontier
`noteResolvedSpan` now gates the non-boundary flush on `frontierFlushLimiter.canSave`, and `canSave` returns false whenever `interval ‹= 0`. `ca.flushFrequency` is 0 when the user sets `min_checkpoint_frequency='0s'`. The `aggregatorFlushJitter` setting description (line 742) still treats 0 as a valid value, and the old code flushed on every advance in that case. With the change, a changefeed using `min_checkpoint_frequency='0s'` only sends frontier updates to the change frontier on a boundary (`forceFlush`). Progress and resolved timestamps then stall for non-boundary spans. Treating a zero interval as "always allow" for this limiter, or special-casing 0 in the aggregator, would keep the old behavior. The changed test in show_changefeed_jobs_test.go, which switches from '0s' to '1s', suggests the regression was worked around rather than fixed.
[P2] Backfill and lagging-span checkpoint flush path removed from the aggregator
The old `checkpointSpans` branch is gone. It flushed the aggregator frontier to the change frontier on the `span_checkpoint.interval` cadence when `InBackfill` or `HasLaggingSpans` was true, even when the min_checkpoint_frequency window had not elapsed. Now the aggregator pushes updates only on the min_checkpoint_frequency cadence (30s by default, or whatever the user set). If `span_checkpoint.interval` is shorter than min_checkpoint_frequency, for example the roachtest's 1s or 5s, span checkpoints are written less often than configured. A restart during a long backfill therefore re-scans more data. If this isn't intentional, the setting no longer takes effect on the aggregator side.
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d1 | high | ✗ | — |
canSave draws a new jitter on every call. The removed nextFlushWithJitter computed a single deadline once after each flush. Now the aggregator polls on every resolved span, so once the base interval has passed, any small draw lets the save through. Flushes therefore cluster near the lower bound and the configured staggering is largely lost. [grounded: diff] [consensus 3/3: valid=3]
The finding is about weak test coverage: the new roachtest variant only checks the unique row count and just logs duplicates. Production code does not misbehave because of this, so it is a test-adequacy note rather than a defect. [grounded: tree pkg/cmd/roachtest/tests/cdc.go] [consensus 3/3: minor=3]
raw comments (4)
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
1\. Busy changefeeds lose flush jitter <code>🐞 Bug</code> <code>➹ Performance</code>
<pre>
<b><i>saveRateLimiter.canSave</i></b> draws a fresh random delay every time it is polled instead of retaining
one deadline until <b><i>doneSave</i></b>. Because every resolved event polls the limiter, busy aggregators make
repeated draws after the base interval and converge on its lower bound, so sink and frontier flushes
happen more frequently and with far less staggering than configured.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
The save limiter resamples its jitter whenever `canSave` is called, allowing frequent resolved events to repeatedly retry until a small jitter is selected. Jitter should define one stable deadline for each interval.
## Fix Focus Areas
- pkg/ccl/changefeedccl/changefeed_processors.go[2393-2425]
- pkg/ccl/changefeedccl/changefeed_processors_test.go[72-103]
## Recommended Fix
Store the next jittered save deadline in `saveRateLimiter`, sample it once after each completed save, and compare subsequent `canSave` calls against that retained deadline. Extend the test to poll repeatedly before the selected deadline and verify that polling cannot change it.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
2\. Frontier persistence goes untested <code>🐞 Bug</code> <code>⚙ Maintainability</code>
<pre>
<b><i>runCDCInitialScanRollingRestart</i></b> configures frontier persistence as the sole fine-grained
checkpoint mechanism, but its only assertion compares unique row IDs and merely logs duplicate
deliveries. If persisted spans are never loaded after rolling restarts, replaying the initial scan
still produces every unique ID and the new variant passes, so regressions in the behavior it was
added to cover remain undetected.
</pre>
<details>
<summary><strong>Agent Prompt</strong></summary>
```
## Issue description
The new frontier-persistence test variant can pass when persisted frontier state is never written or restored because replayed rows still satisfy its unique-row assertion. The test needs direct evidence that progress persisted before a restart is used afterward.
## Fix Focus Areas
- pkg/cmd/roachtest/tests/cdc.go[1121-1126]
- pkg/cmd/roachtest/tests/cdc.go[1147-1170]
- pkg/cmd/roachtest/tests/cdc.go[1219-1230]
## Recommended Fix
Add an observable assertion that frontier entries are persisted before a restart and restored by the replanned flow afterward, using job-frontier state or a dedicated test metric. Keep the unique-row check as the end-to-end correctness assertion rather than treating duplicate logging as proof of recovery.
```
<code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
</details>
<h3>PR Summary by Qodo</h3>
Unify changefeed frontier checkpoint rate limiting
<code>🐞 Bug fix</code> <code>🧪 Tests</code> <code>🕐 40+ Minutes</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<details>
<summary>AI Description</summary>
<dl>
<dd>
<br/>
><pre>
>• Route aggregator frontier flushes through shared duration-aware rate limiting.
>• Persist full frontier progress during backfills and rolling node restarts.
>• Add unit and nightly coverage for checkpoint timing and recovery.
></pre>
</dd>
</dl>
</details>
<details>
<summary>Diagram</summary>
<dl>
<dd>
<br/>
```mermaid
sequenceDiagram
participant KV as KV Feed
participant Agg as Aggregator
participant AFL as Flush Limiter
participant Sink as Event Sink
participant Front as Frontier
participant PFL as Persist Limiter
participant Store as Frontier Store
KV->>Agg: Resolved span
Agg->>AFL: Check interval
AFL-->>Agg: Allow with jitter
Agg->>Sink: Flush events
Agg->>Front: Emit full frontier
Front->>PFL: Check persistence
PFL-->>Front: Allow by duration
Front->>Store: Persist frontier
```
</dd>
</dl>
</details>
<details>
<summary>High-Level Assessment</summary>
<dl>
<dd>
<br/>
>The following are alternative approaches to this PR:
<details>
<summary><b>1. Retain separate flush schedules</b></summary>
<dl>
<dd>
- ➕ Smaller change to existing aggregator checkpoint logic
- ➕ Preserves independently tailored scheduling paths
- ➖ Duplicates interval and jitter handling
- ➖ Leaves aggregator and coordinator throttling behavior inconsistent
- ➖ Provides weaker testability without an injectable clock
</dd>
</dl>
</details>
<details>
<summary><b>2. Use a periodic flush goroutine</b></summary>
<dl>
<dd>
- ➕ Can flush on wall-clock intervals without waiting for resolved spans
- ➕ Separates scheduling from span processing
- ➖ Adds processor lifecycle and synchronization complexity
- ➖ Creates more concurrency around sink and frontier state
- ➖ Resolved spans already arrive periodically under normal operation
</dd>
</dl>
</details>
>**Recommendation:** Use the PR's shared, configurable limiter abstraction with separate instances for aggregator flushing and coordinator persistence. It centralizes interval, jitter, adaptive save-duration, validation, and clock behavior without introducing new processor concurrency.
</dd>
</dl>
</details>
<details>
<summary> Files changed (7) <code> +272 / -216 </code> </summary>
<dl>
<dd>
<br/>
<details>
<summary>Bug fix (1) <code> +81 / -63 </code></summary>
<dl>
<dd>
<details>
<summary>changefeed_processors.go<code>Unify frontier flush and persistence throttling</code> <code>+81/-63</code></summary>
<br/>
>Unify frontier flush and persistence throttling
>
><pre>
>• Replaces aggregator-specific flush timestamps and span flush logic with configurable save-rate limiters supporting dynamic intervals, optional jitter, validation, and injectable clocks. Aggregators now emit full frontiers through the limiter, while the coordinator uses the same abstraction for durable frontier persistence.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-ec033be3bf0ce6dd0ca1f16617e9cb9f183bf4b8e9dfc3bf1cb49403e1893e7e'>pkg/ccl/changefeedccl/changefeed_processors.go</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Tests (5) <code> +190 / -153 </code></summary>
<dl>
<dd>
<details>
<summary>alter_changefeed_test.go<code>Allow full-table resolved spans in backfill test</code> <code>+1/-7</code></summary>
<br/>
>Allow full-table resolved spans in backfill test
>
><pre>
>• Updates the backfill target-addition fixture to pass through the complete table span instead of mutating and suppressing it. This aligns the test with full-frontier aggregator flushing.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-1216dc557ecd09f5dee692c75f87364cb6ec46065738b2f0facf2918a21c4fa0'>pkg/ccl/changefeedccl/alter_changefeed_test.go</a>
</details>
<details>
<summary>changefeed_processors_test.go<code>Test configurable save-rate limiting</code> <code>+154/-0</code></summary>
<br/>
>Test configurable save-rate limiting
>
><pre>
>• Adds deterministic manual-clock coverage for positive, disabled, and tiny intervals; jitter behavior; adaptive save-duration throttling; and invalid limiter configurations.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-688d53a724c1cca0df940b7372770a6b1a713e342e5ab99dbc7d354654a7d2d6'>pkg/ccl/changefeedccl/changefeed_processors_test.go</a>
</details>
<details>
<summary>changefeed_test.go<code>Adapt checkpoint tests to limiter semantics</code> <code>+1/-134</code></summary>
<br/>
>Adapt checkpoint tests to limiter semantics
>
><pre>
>• Sets an explicit rapid checkpoint frequency for the backfill recovery test. Removes the old standalone flush-jitter test because jitter is now covered through the save-rate limiter.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-5a691422b367ab03b850a911a237faed7ee664fa6e15d4c1f2f9cd3123819fb5'>pkg/ccl/changefeedccl/changefeed_test.go</a>
</details>
<details>
<summary>show_changefeed_jobs_test.go<code>Use an enabled checkpoint interval for high-water test</code> <code>+1/-1</code></summary>
<br/>
>Use an enabled checkpoint interval for high-water test
>
><pre>
>• Changes the feed to use resolved timestamps with a one-second minimum checkpoint frequency, ensuring high-water progress is emitted now that non-positive intervals disable saves.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-50eb01e0fb6241aac76b1fe27c506b1959dd2e4d851047e59cd0b8279dc795c5'>pkg/ccl/changefeedccl/show_changefeed_jobs_test.go</a>
</details>
<details>
<summary>cdc.go<code>Add frontier-persistence rolling restart coverage</code> <code>+33/-11</code></summary>
<br/>
>Add frontier-persistence rolling restart coverage
>
><pre>
>• Adds a nightly rolling-restart scenario that relies on coordinator frontier persistence while disabling span and shutdown checkpoints. Expands and scatters table ranges across all checkpoint modes and configures explicit checkpoint settings for isolation.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-7276eae775645763129d898332c5c5b40465682ae914ff892ea48d2efe89532a'>pkg/cmd/roachtest/tests/cdc.go</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Other (1) <code> +1 / -0 </code></summary>
<dl>
<dd>
<details>
<summary>BUILD.bazel<code>Register save-rate limiter tests</code> <code>+1/-0</code></summary>
<br/>
>Register save-rate limiter tests
>
><pre>
>• Adds the new changefeed processor test file to the Bazel go_test target.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-981c86d1c6590b9e7c4934823df5e388b51db43599dced882e204f5c71ad7401'>pkg/ccl/changefeedccl/BUILD.bazel</a>
</details>
</dd>
</dl>
</details>
</dd>
</dl>
</details>
<h3>Code Review by Qodo</h3>
<code>🐞 Bugs (2)</code> <code>📘 Rule violations (0)</code> <code>📜 Skill insights (0)</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<br/>
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended">
<details>
<summary> 1. Busy changefeeds lose flush jitter <code>🐞 Bug</code> <code>➹ Performance</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
><b><i>saveRateLimiter.canSave</i></b> draws a fresh random delay every time it is polled instead of retaining
>one deadline until <b><i>doneSave</i></b>. Because every resolved event polls the limiter, busy aggregators make
>repeated draws after the base interval and converge on its lower bound, so sink and frontier flushes
>happen more frequently and with far less staggering than configured.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[pkg/ccl/changefeedccl/changefeed_processors.go[R2398-2401]](https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-ec033be3bf0ce6dd0ca1f16617e9cb9f183bf4b8e9dfc3bf1cb49403e1893e7eR2398-R2401)</code>
>
>```diff
>+ if l.config.jitter != nil {
>+ if jitter := l.config.jitter(); jitter > 0 {
>+ if maxJitter := time.Duration(jitter * float64(interval)); maxJitter > 0 {
>+ interval += time.Duration(rand.Int63n(int64(maxJitter) + 1))
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
>Every resolved event reaches <b><i>noteResolvedSpan</i></b>, which invokes <b><i>canSave</i></b>; the limiter then
>independently calls <b><i>rand.Int63n</i></b> on each invocation. The tests advance directly to the maximum
>jitter boundary and therefore do not cover repeated polling within one interval.
></pre>
>
> <code>[pkg/ccl/changefeedccl/changefeed_processors.go[870-883]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_processors.go/#L870-L883)</code>
> <code>[pkg/ccl/changefeedccl/changefeed_processors.go[933-948]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_processors.go/#L933-L948)</code>
> <code>[pkg/ccl/changefeedccl/changefeed_processors.go[2392-2408]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_processors.go/#L2392-L2408)</code>
> <code>[pkg/ccl/changefeedccl/changefeed_processors_test.go[72-85]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_processors_test.go/#L72-L85)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>The save limiter resamples its jitter whenever `canSave` is called, allowing frequent resolved events to repeatedly retry until a small jitter is selected. Jitter should define one stable deadline for each interval.
>
>## Fix Focus Areas
>- pkg/ccl/changefeedccl/changefeed_processors.go[2393-2425]
>- pkg/ccl/changefeedccl/changefeed_processors_test.go[72-103]
>
>## Recommended Fix
>Store the next jittered save deadline in `saveRateLimiter`, sample it once after each completed save, and compare subsequent `canSave` calls against that retained deadline. Extend the test to poll repeatedly before the selected deadline and verify that polling cannot change it.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
<hr/>
</details>
<details>
<summary> 2. Frontier persistence goes untested <code>🐞 Bug</code> <code>⚙ Maintainability</code></summary>
<br/>
> <details open>
><summary>Description</summary>
><br/>
>
><pre>
><b><i>runCDCInitialScanRollingRestart</i></b> configures frontier persistence as the sole fine-grained
>checkpoint mechanism, but its only assertion compares unique row IDs and merely logs duplicate
>deliveries. If persisted spans are never loaded after rolling restarts, replaying the initial scan
>still produces every unique ID and the new variant passes, so regressions in the behavior it was
>added to cover remain undetected.
></pre>
></details>
> <details>
><summary>Code</summary>
><br/>
>
><code>[pkg/cmd/roachtest/tests/cdc.go[R1121-1124]](https://github.com/drape-io/crbench-qodo-2026-09/pull/90/files#diff-7276eae775645763129d898332c5c5b40465682ae914ff892ea48d2efe89532aR1121-R1124)</code>
>
>```diff
>+ case cdcFrontierPersistence:
>+ setupStmts = append(setupStmts,
>+ `SET CLUSTER SETTING changefeed.span_checkpoint.interval = '0'`,
>+ `SET CLUSTER SETTING changefeed.shutdown_checkpoint.enabled = 'false'`,
>```
></details>
> <details>
><summary>Evidence</summary>
><br/>
>
><pre>
>The new mode disables both periodic span checkpoints and shutdown checkpoints, leaving frontier
>persistence as its intended recovery mechanism. Replanning reads persisted spans, but the roachtest
>only checks that all unique IDs eventually arrive; its duplicate count is logged without any
>assertion, so a complete replay satisfies the test.
></pre>
>
> <code>[pkg/cmd/roachtest/tests/cdc.go[1108-1126]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/cmd/roachtest/tests/cdc.go/#L1108-L1126)</code>
> <code>[pkg/cmd/roachtest/tests/cdc.go[1145-1170]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/cmd/roachtest/tests/cdc.go/#L1145-L1170)</code>
> <code>[pkg/cmd/roachtest/tests/cdc.go[1219-1230]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/cmd/roachtest/tests/cdc.go/#L1219-L1230)</code>
> <code>[pkg/ccl/changefeedccl/changefeed_dist.go[474-488]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_dist.go/#L474-L488)</code>
> <code>[pkg/ccl/changefeedccl/changefeed_processors.go[1914-1932]](https://github.com/drape-io/crbench-qodo-2026-09/blob/934fb128538433dac880103df4fdc33c73109958/pkg/ccl/changefeedccl/changefeed_processors.go/#L1914-L1932)</code>
></details>
> <details>
><summary>Agent prompt</summary>
><br/>
>
>```
>The issue below was found during a code review. Follow the provided context and guidance below and implement a solution
>
>## Issue description
>The new frontier-persistence test variant can pass when persisted frontier state is never written or restored because replayed rows still satisfy its unique-row assertion. The test needs direct evidence that progress persisted before a restart is used afterward.
>
>## Fix Focus Areas
>- pkg/cmd/roachtest/tests/cdc.go[1121-1126]
>- pkg/cmd/roachtest/tests/cdc.go[1147-1170]
>- pkg/cmd/roachtest/tests/cdc.go[1219-1230]
>
>## Recommended Fix
>Add an observable assertion that frontier entries are persisted before a restart and restored by the replanned flow afterward, using job-frontier state or a dedicated test metric. Keep the unique-row check as the end-to-end correctness assertion rather than treating duplicate logging as proof of recovery.
>```
> <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code>
></details>
<hr/>
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-context:start -->
<details><summary><strong>Context sources</strong></summary>
<div>Review mode: <code>🧠 Deep</code>: This changes core changefeed checkpointing and rate-limiter behavior across production code, tests, and roachtests, with multiple independent logic paths and timing-sensitive semantics that could hide several subtle regressions.</div>
<!-- qodo-context:end -->
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-daily-tip:start -->
<details>
<summary><strong>Tip of the day</strong></summary>
<br/>
<pre>💡 Did you know, you can reply 'qodo' on any finding to push back, ask questions, or dig deeper</pre>
<a href="https://docs.qodo.ai/tips-and-tricks">More tips ↗</a> | <a href="https://app.qodo.ai/configurations?tab=display-preferences">Customize Qodo ↗</a> | <a href="https://docs.qodo.ai">Qodo docs ↗</a>
</details>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<!-- qodo-daily-tip:end -->
<!-- https://github.com/drape-io/crbench-qodo-2026-09/commit/934fb128538433dac880103df4fdc33c73109958 -->
<a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>