Tombstone updater re-leases its descriptor on every call and reports failed updates as success
Tier 2 easy go · cross-file · real merged PR from cockroachdb/cockroach #143567
Ground-truth defects
getDeleter re-acquires when `!timestamp.After(expiration)` (timestamp <= expiration), which is the lease's valid range, so it re-leases every call while valid and reuses the expired descriptor (setting a past UpdateDeadline) once timestamp > expiration — inverted relative to hasExpired's `expiration.LessEq(timestamp)`.
`updateTombstone` guards with `if err != nil && isLwwLoser(err)` and then re-tests `if isLwwLoser(err)` inside that branch, which is always true, making the `return batchStats{}, err` arm unreachable. The consequence is the fall-through: any genuine error (addToBatch failure, `txn.KV().Run` failure, a lease error) fails the outer condition and reaches `return batchStats{}, nil`, reporting success on a failed tombstone update.
The change under review
diff (truncated — full diff)
@@ -16,6 +16,7 @@ go_library( "purgatory.go", "range_stats.go", "replication_statements.go", + "tombstone_updater.go", "udf_row_processor.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/crosscluster/logical", @@ -154,6 +155,7 @@ go_test( "//pkg/sql/catalog/descpb", "//pkg/sql/catalog/descs", "//pkg/sql/catalog/desctestutils", + "//pkg/sql/catalog/lease", "//pkg/sql/catalog/resolver", "//pkg/sql/execinfra", "//pkg/sql/execinfrapb", @@ -1387,6 +1387,86 @@ func TestThreeWayReplication(t *testing.T) { verifyExpectedRowAllServers(t, runners, expectedRows, dbNames) } +func TestTombstoneUpdate(t *testing.T) { + defer leaktest.AfterTest(t)() + skip.UnderDeadlock(t) + defer log.Scope(t).Close(t) + // This test is a regression test for a bug where replicating a delete does + // not update the origin timestamp assigned to a tombstone if the replicated + // value is overwriting a tombstone. + // + // Replicating the bug requires three databases ('src-a', 'src-b', 'dst'). + // + // Here's the timeline of events: + // + // 1. row is inserted into 'src-a', 'src-b', and 'dst' + // 2. row is deleted from 'dst' + // 3. row is updated in 'src-b' + // 4. row is deleted in 'src-a' + // + // 5. Replication is started from 'src-a' -> 'dst' and allowed to catch up to now. + // 6. Replication is started from 'srb-b' -> 'dst' and allowed to catch up to now. + // + // In this example, the last delete is the LWW winner. But if tombstones are not updated, then the update in 'src-b' will be + // replicated to 'dst' will win LWW since the replicated delete was no-op'd. + + ctx := context.Background() + + clusterArgs := base.TestClusterArgs{ + ServerArgs: base.TestServerArgs{ + DefaultTestTenant: base.TestControlsTenantsExplicitly, + Knobs: base.TestingKnobs{ + JobsTestingKnobs: jobs.NewTestingKnobsWithShortIntervals(), + }, + }, + } + + numDBs := 3 + server, s, runners, dbNames := setupServerWithNumDBs(t, ctx, clusterArgs, 1, numDBs) + defer server.Stopper().Stop(ctx) + PGURLs := GetPGURLs(t, s, dbNames) + + // Insert row into all three databases + for i := range runners { + runners[i].Exec(t, "UPSERT INTO tab VALUES (1, 'initial')") + } + + srcA, srcB, dst := runners[0], runners[1], runners[2] + urlSrcA, urlSrcB, _ := PGURLs[0].String(), PGURLs[1].String(), PGURLs[2].String() + + // Grab timestamp that will be used as a cursor for starting the logical replication jobs + start := s.Clock().Now() + + // Delete the row in the dest, this should lose all LWW conflicts. + dst.Exec(t, "DELETE FROM tab WHERE pk = 1") + + // Update the row in one of the sources, this should lose LWW conlficts to + // the final delete. + srcB.Exec(t, "UPSERT INTO tab VALUES (1, 'updated')") + + // Delete the row in the other source, this should win LWW. + srcA.Exec(t, "DELETE FROM tab WHERE pk = 1") + + dst.CheckQueryResults(t, "SELECT * FROM tab", [][]string{}) + srcA.CheckQueryResults(t, "SELECT * FROM tab", [][]string{}) + srcB.CheckQueryResults(t, "SELECT * FROM tab", [][]string{{"1", "updated"}}) + + // 5. Replicate the delete from 'src-a' -> 'dst' + var jobIDSrcA jobspb.JobID + dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH MODE = VALIDATED, CURSOR = $2", + urlSrcA, start.AsOfSystemTime()).Scan(&jobIDSrcA) + WaitUntilReplicatedTime(t, s.Clock().Now(), dst, jobIDSrcA) + + // 6. Replicate the update from 'src-b' -> 'dst' + var jobIDSrcB jobspb.JobID + dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH MODE = VALIDATED, CURSOR = $2", + urlSrcB, start.AsOfSystemTime()).Scan(&jobIDSrcB) + WaitUntilReplicatedTime(t, s.Clock().Now(), dst, jobIDSrcB) + + // Verify that the delete won LWW since it has the highest mvcc value + dst.CheckQueryResults(t, "SELECT * FROM tab WHERE pk = 1", [][]string{}) +} + func TestForeignKeyConstraints(t *testing.T) { defer leaktest.AfterTest(t)() skip.UnderDeadlock(t) @@ -25,6 +25,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/sql" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/lease" "github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc" "github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" @@ -738,6 +739,8 @@ func (lrw *logicalReplicationWriterProcessor) setupBatchHandlers(ctx context.Con // avoid creating a new copy on each executor usage. flowCtx.Cfg.DB.Executor(isql.WithSessionData(sql.NewInternalSessionData(ctx, flowCtx.Cfg.Settings, "" /* opName */))), sd, lrw.spec, + flowCtx.Codec(), + flowCtx.Cfg.LeaseManager.(*lease.Manager), ) if err != nil { return err @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/lease" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/isql" "github.com/cockroachdb/cockroach/pkg/sql/lexbase" @@ -79,6 +80,7 @@ type querier interface { InsertRow(ctx context.Context, txn isql.Txn, ie isql.Executor, row cdcevent.Row, prevRow *cdcevent.Row, likelyInsert bool) (batchStats, error) DeleteRow(ctx context.Context, txn isql.Txn, ie isql.Executor, row cdcevent.Row, prevRow *cdcevent.Row) (batchStats, error) RequiresParsedBeforeRow(catid.DescID) bool + ReleaseLeases(ctx context.Context) } // isLwwLoser returns true if the error is a ConditionFailedError with an @@ -274,7 +276,9 @@ func (sqlRowProcessor) ReportMutations(_ *stats.Refresher) {} // ReleaseLeases implements the BatchHandler interface but is a no-op since each // query does this itself. -func (sqlRowProcessor) ReleaseLeases(_ context.Context) {} +func (srp *sqlRowProcessor) ReleaseLeases(ctx context.Context) { + srp.querier.ReleaseLeases(ctx) +} func (*sqlRowProcessor) Close(ctx context.Context) {} @@ -458,6 +462,8 @@ func makeSQLProcessor( ie isql.Executor, sd *sessiondata.SessionData, spec execinfrapb.LogicalReplicationWriterSpec, + codec keys.SQLCodec, + leaseMgr *lease.Manager, ) (*sqlRowProcessor, error) { needUDFQuerier := false @@ -469,10 +475,14 @@ func makeSQLProcessor( lwwQuerier := &lwwQuerier{ settings: settings, + codec: codec, + db: db, + leaseMgr: leaseMgr, queryBuffer: queryBuffer{ deleteQueries: make(map[catid.DescID]queryBuilder, len(tableConfigByDestID)), insertQueries: make(map[catid.DescID]map[catid.FamilyID]queryBuilder, len(tableConfigByDestID)), }, + tombstoneUpdaters: make(map[descpb.ID]*tombstoneUpdater, len(tableConfigByDestID)), ieOverrideOptimisticInsert: getIEOverride(replicatedOptimisticInsertOpName, jobID), ieOverrideInsert: getIEOverride(replicatedInsertOpName, jobID), ieOverrideDelete: getIEOverride(replicatedDeleteOpName, jobID), @@ -532,6 +542,13 @@ func (m *muxQuerier) RequiresParsedBeforeRow(id catid.DescID) bool { return m.shouldUseUDF[id] } +func (m *muxQuerier) ReleaseLeases(ctx context.Context) { + if m.udfQuerier != nil { + m.udfQuerier.ReleaseLeases(ctx) + } + m.lwwQuerier.ReleaseLeases(ctx) +} + // lwwQuerier is a querier that implements partial last-write-wins // semantics using SQL queries. // @@ -542,12 +559,16 @@ func (m *muxQuerier) RequiresParsedBeforeRow(id catid.DescID) bool { // // See the design document for possible solutions to these problems. type lwwQuerier struct { - settings *cluster.Settings - queryBuffer queryBuffer + settings *cluster.Settings + codec keys.SQLCodec + db descs.DB + queryBuffer queryBuffer + tombstoneUpdaters map[descpb.ID]*tombstoneUpdater ieOverrideOptimisticInsert sessiondata.InternalExecutorOverride ieOverrideInsert sessiondata.InternalExecutorOverride ieOverrideDelete sessiondata.InternalExecutorOverride + leaseMgr *lease.Manager } func (lww *lwwQuerier) AddTable(targetDescID int32, tc sqlProcessorTableConfig) error { @@ -561,6 +582,15 @@ func (lww *lwwQuerier) AddTable(targetDescID int32, tc sqlProcessorTableConfig) if err != nil { return err } + + lww.tombstoneUpdaters[td.GetID()] = newTombstoneUpdater( + lww.codec, + lww.db.KV(), + lww.leaseMgr, + catid.DescID(targetDescID), + lww.settings, + ) + return nil } @@ -665,13 +695,26 @@ func (lww *lwwQuerier) DeleteRow( sess.QualityOfService = nil } sess.OriginTimestampForLogicalDataReplication = row.MvccTimestamp - if _, err := ie.ExecParsed(ctx, replicatedDeleteOpName, kvTxn, sess, stmt, datums...); err != nil { + rowCount, err := ie.ExecParsed(ctx, replicatedDeleteOpName, kvTxn, sess, stmt, datums...) + if err != nil { log.Warningf(ctx, "replicated delete failed (query: %s): %s", stmt.SQL, err.Error()) return batchStats{}, err } + if rowCount != 1 { + // NOTE: at this point we don't know if we are updating a tombstone or if + // we are losing LWW. As long as it is a LWW loss or a tombstone update, + // updateTombstone will return okay. + return lww.tombstoneUpdaters[row.TableID].updateTombstone(ctx, txn, row) + } return batchStats{}, nil } +func (lww *lwwQuerier) ReleaseLeases(ctx context.Context) { + for _, tu := range lww.tombstoneUpdaters { + tu.ReleaseLeases(ctx) + } +} + const ( insertQueryOptimistic = `INSERT INTO [%d AS t] (%s) VALUES (%s)` insertQueryPessimistic = ` @@ -20,6 +20,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog/descs" "github.com/cockroachdb/cockroach/pkg/sql/catalog/desctestutils" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/lease" "github.com/cockroachdb/cockroach/pkg/sql/execinfra" "github.com/cockroachdb/cockroach/pkg/sql/execinfrapb" "github.com/cockroachdb/cockroach/pkg/sql/isql" @@ -79,7 +80,7 @@ func TestLWWInsertQueryGeneration(t *testing.T) { return tableName } - setup := func(t *testing.T, schemaTmpl string) (*sqlRowProcessor, func(...interface{}) roachpb.KeyValue) { + setup := func(t *testing.T, schemaTmpl string) (*sqlRowProcessor, func(...interface{}) roachpb.KeyValue, func()) { tableNameSrc := createTable(t, schemaTmpl) tableNameDst := createTable(t, schemaTmpl) srcDesc := desctestutils.TestingGetPublicTableDescriptor(s.DB(), s.Codec(), "defaultdb", tableNameSrc) @@ -89,20 +90,23 @@ func TestLWWInsertQueryGeneration(t *testing.T) { dstDesc.GetID(): { srcDesc: srcDesc, }, - }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{}) + }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{}, s.Codec(), s.LeaseManager().(*lease.Manager)) require.NoError(t, err) return rp, func(datums ...interface{}) roachpb.KeyValue { - kv := replicationtestutils.EncodeKV(t, s.Codec(), srcDesc, datums...) - kv.Value.Timestamp = hlc.Timestamp{WallTime: timeutil.Now().UnixNano()} - return kv - } + kv := replicationtestutils.EncodeKV(t, s.Codec(), srcDesc, datums...) + kv.Value.Timestamp = hlc.Timestamp{WallTime: timeutil.Now().UnixNano()} + return kv + }, func() { + rp.ReleaseLeases(ctx) + } } for _, tc := range testCases { t.Run(fmt.Sprintf("%s/insert", tc.name), func(t *testing.T) { runner.Exec(t, "SET CLUSTER SETTING logical_replication.consumer.try_optimistic_insert.enabled=true") defer runner.Exec(t, "RESET CLUSTER SETTING logical_replication.consumer.try_optimistic_insert.enabled") - rp, encoder := setup(t, tc.schemaTmpl) + rp, encoder, cleanup := setup(t, tc.schemaTmpl) + defer cleanup() keyValue := encoder(tc.row...) require.NoError(t, s.InternalDB().(isql.DB).Txn(ctx, func(ctx context.Context, txn isql.Txn) error { _, err := rp.ProcessRow(ctx, txn, keyValue, roachpb.Value{}) @@ -112,7 +116,8 @@ func TestLWWInsertQueryGeneration(t *testing.T) { t.Run(fmt.Sprintf("%s/insert-without-optimistic-insert", tc.name), func(t *testing.T) { runner.Exec(t, "SET CLUSTER SETTING logical_replication.consumer.try_optimistic_insert.enabled=false") defer runner.Exec(t, "RESET CLUSTER SETTING logical_replication.consumer.try_optimistic_insert.enabled") - rp, encoder := setup(t, tc.schemaTmpl) + rp, encoder, cleanup := setup(t, tc.schemaTmpl) + defer cleanup() keyValue := encoder(tc.row...) require.NoError(t, s.InternalDB().(isql.DB).Txn(ctx, func(ctx context.Context, txn isql.Txn) error { _, err := rp.ProcessRow(ctx, txn, keyValue, roachpb.Value{}) @@ -120,7 +125,8 @@ func TestLWWInsertQueryGeneration(t *testing.T) { })) }) t.Run(fmt.Sprintf("%s/delete", tc.name), func(t *testing.T) { - rp, encoder := setup(t, tc.schemaTmpl) + rp, encoder, cleanup := setup(t, tc.schemaTmpl) + defer cleanup() keyValue := encoder(tc.row...) keyValue.Value.RawBytes = nil require.NoError(t, s.InternalDB().(isql.DB).Txn(ctx, func(ctx context.Context, txn isql.Txn) error { @@ -157,7 +163,7 @@ func BenchmarkLWWInsertBatch(b *testing.B) { desc.GetID(): { srcDesc: desc, }, - }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalDB().(isql.DB).Executor(isql.WithSessionData(sd)), sd, execinfrapb.LogicalReplicationWriterSpec{}) + }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalDB().(isql.DB).Executor(isql.WithSessionData(sd)), sd, execinfrapb.LogicalReplicationWriterSpec{}, s.Codec(), s.LeaseManager().(*lease.Manager)) require.NoError(b, err) // In some configs, we'll be simulating processing the same INSERT over and @@ -345,7 +351,7 @@ func TestLWWConflictResolution(t *testing.T) { dstDesc.GetID(): { srcDesc: srcDesc, }, - }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{}) + }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{}, s.Codec(), s.LeaseManager().(*lease.Manager)) require.NoError(t, err) if useKVProc { new file mode 100644 @@ -0,0 +1,184 @@ +// Copyright 2025 The Cockroach Authors. +// +// Use of this software is governed by the CockroachDB Software License +// included in the /LICENSE file. + +package logical + +import ( + "context" + + "github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdcevent" + "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/kv" + "github.com/cockroachdb/cockroach/pkg/settings/cluster" + "github.com/cockroachdb/cockroach/pkg/sql/catalog" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog/lease" + "github.com/cockroachdb/cockroach/pkg/sql/isql" + "github.com/cockroachdb/cockroach/pkg/sql/row" + "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" +) + +// tombstoneUpdater is a helper for updating the mvcc origin timestamp assigned +// to a tombstone. It internally manages leasing a descriptor for a single +// table. The lease should be released by calling ReleaseLeases. +type tombstoneUpdater struct { + codec keys.SQLCodec + db *kv.DB + leaseMgr *lease.Manager + settings *cluster.Settings + descID descpb.ID + + // leased holds fields whose lifetimes are tied to a leased descriptor. + leased struct { + // descriptor is a leased descriptor. Callers should use getDeleter to + // ensure the lease is valid for the current transaction. + descriptor lease.LeasedDescriptor + // deleter is a row.Deleter that uses the leased descriptor. Callers should + // use getDeleter to ensure the lease is valid for the current transaction. + deleter row.Deleter + // columns are the name of the columns that are expected in the cdc row. + columns []string + } + + // scratch is a scratch buffer for the tombstone updater. This is reused + // across calls to addToBatch to avoid allocations. + scratch []tree.Datum +} + +func (c *tombstoneUpdater) ReleaseLeases(ctx context.Context) { + // NOTE: ReleaseLeases may be called multiple times since its called if the lease + // expires and a new lease is acquired. + if c.leased.descriptor != nil { + c.leased.descriptor.Release(ctx) + c.leased.descriptor = nil + c.leased.deleter = row.Deleter{} + c.leased.columns = c.leased.columns[:0] + } +} + +func newTombstoneUpdater( + codec keys.SQLCodec, + db *kv.DB, + leaseMgr *lease.Manager, + descID descpb.ID, + settings *cluster.Settings, +) *tombstoneUpdater { + return &tombstoneUpdater{ + codec: codec, + db: db, + leaseMgr: leaseMgr, + descID: descID, + settings: settings, + } +} + +// updateTombstone attempts to update the tombstone for the given row. This is +// expected to always succeed. The delete will only return zero rows if the +// operation loses LWW or the row does not exist. So if the cput fails on a +// condition, it should also fail on LWW, which is treated as a success. +func (tu *tombstoneUpdater) updateTombstone( + ctx context.Context, txn isql.Txn, afterRow cdcevent.Row, +) (batchStats, error) { + err := func() error { + if txn != nil { + // If updateTombstone is called in a transaction, create and run a batch + // in the transaction. + batch := txn.KV().NewBatch() + if err := tu.addToBatch(ctx, txn.KV(), batch, afterRow); err != nil { + return err + } + return txn.KV().Run(ctx, batch) + } + // If updateTombstone is called outside of a transaction, create and run a + // 1pc transaction. + return tu.db.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error { + batch := txn.NewBatch() + if err := tu.addToBatch(ctx, txn, batch, afterRow); err != nil { + return err + } + return txn.CommitInBatch(ctx, batch) + }) + }() + if err != nil && isLwwLoser(err) { + if isLwwLoser(err) { + return batchStats{kvWriteTooOld: 1}, nil + } + return batchStats{}, err + } + return batchStats{}, nil +} + +func (tu *tombstoneUpdater) addToBatch( + ctx context.Context, txn *kv.Txn, batch *kv.Batch, afterRow cdcevent.Row, +) error { + deleter, err := tu.getDeleter(ctx, txn) + if err != nil { + return err + } + + tu.scratch = tu.scratch[:0] + + // Note that the columns in the cdcevent row are decoded using a descriptor + // for the source column, whereas the row.Deleter column list is initialized + // using columns from the destination table. This is funky, but it works + // because we validate LDR schemas are compatible. + + datums, err := afterRow.DatumsNamed(tu.leased.columns) + if err != nil { + return err + } + if err := datums.Datum(func(d tree.Datum, col cdcevent.ResultColumn) error { + tu.scratch = append(tu.scratch, d) + return nil + }); err != nil { + return err + } + + // the index helpers are never really used since we are always updating a + // tombstone. + var ph row.PartialIndexUpdateHelper
Leaderboard for this case
easy · 7 of 9 bots fully caught it (78% solve rate) · 83% avg coverage.
| # | Bot | Result | Caught | Coverage | Findings | False alarms |
|---|---|---|---|---|---|---|
| 1 | cursor | solved | 2/2 | 100.0% | 3 | 0 |
| 2 | Drape (Deep Review) | solved | 2/2 | 100.0% | 5 | 0 |
| 3 | Drape (Review) | solved | 2/2 | 100.0% | 5 | 0 |
| 4 | greptile | solved | 2/2 | 100.0% | 2 | 0 |
| 5 | Mjolnir (Opus 5.5) | solved | 2/2 | 100.0% | 7 | 0 |
| 6 | Mjolnir (Sonnet 5.5) | solved | 2/2 | 100.0% | 5 | 0 |
| 7 | Claude Code (Opus 5.5) | solved | 2/2 | 100.0% | 3 | 1 |
| 8 | Cubic | partial | 1/2 | 50.0% | 4 | 0 |
| 9 | qodo | miss | 0/2 | 0.0% | 0 | 0 |
Bot results
Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | exact |
| d3 | high | ✓ | exact |
The error path the finding relies on can't be reached: `writeableColunms` in lww_kv_processor.go never fails. Its only return is `return ret, nil`, so the half-initialized lease state the finding describes can't happen today. The lease would also still be released later by ReleaseLeases. [invalid: contradiction] [consensus 3/3: invalid=3]
raw comments (3)
The error handling only returns an error when it is an LWW loss: `if err != nil && isLwwLoser(err) { if isLwwLoser(err) {...}; return batchStats{}, err }`. The inner `return err` can never run, so every other error (lease acquisition failure, decode/DatumsNamed failure, KV errors, retryable txn errors, a deadline exceeded from UpdateDeadline) is swallowed and reported as success. The tombstone origin timestamp then goes un-updated with no error, which is the LWW bug this change is meant to fix, and the event is never retried or sent to purgatory. The outer condition should be `if err != nil`.
The lease-validity check is inverted. `tu.leased.descriptor == nil || !timestamp.After(expiration)` re-acquires when the current lease is still valid (timestamp <= expiration) and keeps the cached lease once it has expired. So every tombstone update releases and re-acquires the lease (extra cost and lease churn), and an expired lease keeps being used: rows get encoded with a possibly stale descriptor, and `txn.UpdateDeadline` is called with a deadline already in the past, so the transaction fails at commit. The condition should be `timestamp.After(expiration)` (or `!timestamp.Less(expiration)`).
If `writeableColunms` fails after `leaseMgr.Acquire` succeeds, `tu.leased.descriptor` stays set but `columns` and `deleter` are left empty or zero. The early return also skips releasing the lease, so a half-initialized state can be reused until the next ReleaseLeases call.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✗ | — |
| d3 | high | ✓ | exact |
The new tombstone updater takes descriptor leases, but only checkpoint() calls ReleaseLeases. sqlRowProcessor.Close is a no-op, so leases taken after the last checkpoint are not released at shutdown. kvRowProcessor.Close releases its leases, so the new code is inconsistent with its sibling. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 5/5: valid=5]
TestLWWConflictResolution runs deletes that affect zero rows (e.g. remote-delete-after-local-delete), which calls updateTombstone and takes a lease that is never released. The benchmark only inserts, so that part of the claim is overstated. Low-impact test hygiene issue. [grounded: tree pkg/crosscluster/logical/lww_row_processor_test.go] [consensus 5/5: valid=5]
The typo in the comment is real, but it is cosmetic and does not change behaviour. [grounded: diff] [consensus 5/5: minor=5]
raw comments (5)
<!-- cubic:v=8855fe9f-e63b-47d2-8fcf-bc62a3e6e9b3 -->
<!-- cubic:review-run=a2a053f9-e531-51ce-8453-2a8936eb1629 -->
<!-- metadata:{"confidence":10} -->
P1: When `addToBatch`, the KV batch, or the 1PC commit fails with a non-LWW error, this condition skips the nested block and returns success, so the caller can acknowledge a tombstone that was never written. Check `err != nil` first, then treat only `isLwwLoser(err)` as success and return other errors.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At pkg/crosscluster/logical/tombstone_updater.go, line 104:
<comment>When `addToBatch`, the KV batch, or the 1PC commit fails with a non-LWW error, this condition skips the nested block and returns success, so the caller can acknowledge a tombstone that was never written. Check `err != nil` first, then treat only `isLwwLoser(err)` as success and return other errors.</comment>
<file context>
@@ -0,0 +1,184 @@
+ return txn.CommitInBatch(ctx, batch)
+ })
+ }()
+ if err != nil && isLwwLoser(err) {
+ if isLwwLoser(err) {
+ return batchStats{kvWriteTooOld: 1}, nil
</file context>
```
</details>
<!-- cubic:v=700ffd10-9710-4a50-bfe0-81fc77339cb3 -->
<!-- cubic:review-run=a2a053f9-e531-51ce-8453-2a8936eb1629 -->
<!-- metadata:{"confidence":9} -->
P2: When the SQL processor closes before the next checkpoint, a tombstone update may have acquired a descriptor lease that shutdown never releases. Release the querier's leases from `Close` as well as checkpoint cleanup so schema changes are not blocked until lease expiration.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At pkg/crosscluster/logical/lww_row_processor.go, line 586:
<comment>When the SQL processor closes before the next checkpoint, a tombstone update may have acquired a descriptor lease that shutdown never releases. Release the querier's leases from `Close` as well as checkpoint cleanup so schema changes are not blocked until lease expiration.</comment>
<file context>
@@ -561,6 +582,15 @@ func (lww *lwwQuerier) AddTable(targetDescID int32, tc sqlProcessorTableConfig)
return err
}
+
+ lww.tombstoneUpdaters[td.GetID()] = newTombstoneUpdater(
+ lww.codec,
+ lww.db.KV(),
</file context>
```
</details>
<!-- cubic:v=932dba6e-3ffa-4655-983d-bb1ff68dc7f8 -->
<!-- cubic:review-run=a2a053f9-e531-51ce-8453-2a8936eb1629 -->
<!-- metadata:{"confidence":7} -->
P3: makeSQLProcessor now holds a descriptor lease per destination table (the tombstone updater acquires one during row processing, released only via ReleaseLeases). TestLWWInsertQueryGeneration added a `cleanup()` that calls rp.ReleaseLeases, but the other two call sites changed to pass s.LeaseManager() — BenchmarkLWWInsertBatch and TestLWWConflictResolution's setup — never call ReleaseLeases, so each sqlRowProcessor leaks its leased descriptor until the server stops. This is inconsistent with the cleanup added here and, in TestLWWConflictResolution (which builds a fresh processor per subtest inside RunTrueAndFalse loops), accumulates leases on the shared test server. Add a deferred ReleaseLeases in both.
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At pkg/crosscluster/logical/lww_row_processor_test.go, line 93:
<comment>makeSQLProcessor now holds a descriptor lease per destination table (the tombstone updater acquires one during row processing, released only via ReleaseLeases). TestLWWInsertQueryGeneration added a `cleanup()` that calls rp.ReleaseLeases, but the other two call sites changed to pass s.LeaseManager() — BenchmarkLWWInsertBatch and TestLWWConflictResolution's setup — never call ReleaseLeases, so each sqlRowProcessor leaks its leased descriptor until the server stops. This is inconsistent with the cleanup added here and, in TestLWWConflictResolution (which builds a fresh processor per subtest inside RunTrueAndFalse loops), accumulates leases on the shared test server. Add a deferred ReleaseLeases in both.</comment>
<file context>
@@ -89,20 +90,23 @@ func TestLWWInsertQueryGeneration(t *testing.T) {
srcDesc: srcDesc,
},
- }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{})
+ }, jobspb.JobID(1), s.InternalDB().(descs.DB), s.InternalExecutor().(isql.Executor), sd, execinfrapb.LogicalReplicationWriterSpec{}, s.Codec(), s.LeaseManager().(*lease.Manager))
require.NoError(t, err)
return rp, func(datums ...interface{}) roachpb.KeyValue {
</file context>
```
</details>
<!-- cubic:v=1fb98250-ab3c-4451-833d-8e9cb6584bf3 -->
<!-- cubic:review-run=a2a053f9-e531-51ce-8453-2a8936eb1629 -->
<!-- metadata:{"confidence":10} -->
P3: Step 6 of the timeline comment misspells the source database as 'srb-b' instead of 'src-b'. Fix the typo so the documented scenario matches the rest of the comment block (line 1460 uses 'src-b' correctly).
<details>
<summary>Prompt for AI agents</summary>
```text
Check if this issue is valid — if so, understand the root cause and fix it. At pkg/crosscluster/logical/logical_replication_job_test.go, line 1408:
<comment>Step 6 of the timeline comment misspells the source database as 'srb-b' instead of 'src-b'. Fix the typo so the documented scenario matches the rest of the comment block (line 1460 uses 'src-b' correctly).</comment>
<file context>
@@ -1387,6 +1387,86 @@ func TestThreeWayReplication(t *testing.T) {
+ // 4. row is deleted in 'src-a'
+ //
+ // 5. Replication is started from 'src-a' -> 'dst' and allowed to catch up to now.
+ // 6. Replication is started from 'srb-b' -> 'dst' and allowed to catch up to now.
+ //
+ // In this example, the last delete is the LWW winner. But if tombstones are not updated, then the update in 'src-b' will be
</file context>
```
</details>
```suggestion
// 6. Replication is started from 'src-b' -> 'dst' and allowed to catch up to now.
```
<!-- cubic:review-summary:start -->
**4 issues found** across 7 files
<!-- cubic:review-summary:end -->
<details>
<summary>Prompt for AI agents (unresolved issues)</summary>
```text
Check if these issues are valid — if so, understand the root cause of each and fix them. If appropriate, use sub-agents to investigate and fix each issue separately.
<file name="pkg/crosscluster/logical/tombstone_updater.go">
<violation number="1" location="pkg/crosscluster/logical/tombstone_updater.go:104">
P1: When `addToBatch`, the KV batch, or the 1PC commit fails with a non-LWW error, this condition skips the nested block and returns success, so the caller can acknowledge a tombstone that was never written. Check `err != nil` first, then treat only `isLwwLoser(err)` as success and return other errors.</violation>
</file>
<file name="pkg/crosscluster/logical/lww_row_processor_test.go">
<violation number="1" location="pkg/crosscluster/logical/lww_row_processor_test.go:93">
P3: makeSQLProcessor now holds a descriptor lease per destination table (the tombstone updater acquires one during row processing, released only via ReleaseLeases). TestLWWInsertQueryGeneration added a `cleanup()` that calls rp.ReleaseLeases, but the other two call sites changed to pass s.LeaseManager() — BenchmarkLWWInsertBatch and TestLWWConflictResolution's setup — never call ReleaseLeases, so each sqlRowProcessor leaks its leased descriptor until the server stops. This is inconsistent with the cleanup added here and, in TestLWWConflictResolution (which builds a fresh processor per subtest inside RunTrueAndFalse loops), accumulates leases on the shared test server. Add a deferred ReleaseLeases in both.</violation>
</file>
<file name="pkg/crosscluster/logical/lww_row_processor.go">
<violation number="1" location="pkg/crosscluster/logical/lww_row_processor.go:586">
P2: When the SQL processor closes before the next checkpoint, a tombstone update may have acquired a descriptor lease that shutdown never releases. Release the querier's leases from `Close` as well as checkpoint cleanup so schema changes are not blocked until lease expiration.</violation>
</file>
<file name="pkg/crosscluster/logical/logical_replication_job_test.go">
<violation number="1" location="pkg/crosscluster/logical/logical_replication_job_test.go:1408">
P3: Step 6 of the timeline comment misspells the source database as 'srb-b' instead of 'src-b'. Fix the typo so the documented scenario matches the rest of the comment block (line 1460 uses 'src-b' correctly).</violation>
</file>
```
</details>
<sub>Reply with feedback, questions, or to request a fix.<br /><br />[Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/89/ai_pr_review_github_delivery_91eb0748538aee586b96d6fa0a61eed07b0fb984eb854ad912dfbf37f4eddc81?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F89)</sub>
<!-- cubic:review-post:ai_pr_review_github_delivery_91eb0748538aee586b96d6fa0a61eed07b0fb984eb854ad912dfbf37f4eddc81:12960305d15c5e7cf75695ba06059a34e8fd4f27:a2a053f9-e531-51ce-8453-2a8936eb1629 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | exact |
| d3 | high | ✓ | exact |
The change makes sqlRowProcessor hold descriptor leases through tombstoneUpdater, but its Close is still a no-op. The processor's close() and setupBatchHandlers only call b.Close and never call ReleaseLeases, so leases acquired since the last checkpoint stay held until they expire. The sibling kvRowProcessor.Close does call ReleaseLeases. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 5/5: valid=5]
raw comments (4)
### Tombstone update swallows non-LWW errors
**High Severity**
<!-- DESCRIPTION START -->
The outer condition requires `isLwwLoser`, so the inner branch is unreachable and any non-LWW error from the tombstone write is discarded. Failed tombstone updates then look successful, which can leave stale origin timestamps and let later writes win last-write-wins incorrectly.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: e2bb9364-5377-412b-8737-b4c732a746d6 -->
<!-- LOCATIONS START
pkg/crosscluster/logical/tombstone_updater.go#L103-L110
LOCATIONS END -->
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmY1YTEzNGEwLTc3ZjQtNDViNS1hNDNkLThhOGM4MTFhMzIzOSIsImVuY3J5cHRpb25LZXkiOiJqbFZ1QjVqcTVua2Z3YVBZXzZyR2NkVWNZenN5UElIYk02OERtSGh2S3ZFIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmY1YTEzNGEwLTc3ZjQtNDViNS1hNDNkLThhOGM4MTFhMzIzOSIsImVuY3J5cHRpb25LZXkiOiJqbFZ1QjVqcTVua2Z3YVBZXzZyR2NkVWNZenN5UElIYk02OERtSGh2S3ZFIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI4LCJjb21taXRTaGEiOiJiOWRhYmZjN2M3NTdjZGEyOWE5MDUzODJkOGY3MjdjYzUwM2NhY2RjIiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit b9dabfc7c757cda29a905382d8f727cc503cacdc. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
### Lease expiration check is inverted
**High Severity**
<!-- DESCRIPTION START -->
`getDeleter` re-acquires when the transaction timestamp is still before lease expiration and reuses the lease after it has expired. The arguments to `After` are swapped relative to the KV processor, so expired descriptors can be used and `UpdateDeadline` can be called with a past timestamp.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: cc8536f0-bf74-4369-9389-191920366b43 -->
<!-- LOCATIONS START
pkg/crosscluster/logical/tombstone_updater.go#L159-L160
LOCATIONS END -->
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmMzODc0NGE5LWVhMGYtNGM1Yi04Njc2LWNkODFiN2VjYjRjOSIsImVuY3J5cHRpb25LZXkiOiJySXNhNFRSSGx1QzE4LXNxVVFLTl9XaWdXOUU2Tm1IZ2RpWEt3NDJMS19zIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmMzODc0NGE5LWVhMGYtNGM1Yi04Njc2LWNkODFiN2VjYjRjOSIsImVuY3J5cHRpb25LZXkiOiJySXNhNFRSSGx1QzE4LXNxVVFLTl9XaWdXOUU2Tm1IZ2RpWEt3NDJMS19zIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI4LCJjb21taXRTaGEiOiJiOWRhYmZjN2M3NTdjZGEyOWE5MDUzODJkOGY3MjdjYzUwM2NhY2RjIiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit b9dabfc7c757cda29a905382d8f727cc503cacdc. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
### Close skips releasing held leases
**Medium Severity**
<!-- DESCRIPTION START -->
`sqlRowProcessor` now holds descriptor leases via `tombstoneUpdater`, but `Close` is still a no-op. When batch handlers are recreated or the processor shuts down, those leases stay held until they expire, which can delay schema changes such as dropping the destination table.
<!-- DESCRIPTION END -->
<!-- BUGBOT_BUG_ID: 3ed2c5c7-551c-4322-bf32-31a1ba50e3de -->
<!-- LOCATIONS START
pkg/crosscluster/logical/lww_row_processor.go#L282-L283
LOCATIONS END -->
<div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjYzMjJmYjU5LTQxOWMtNDg1Mi1iYjZkLWY3MzE5NTllZTc2OSIsImVuY3J5cHRpb25LZXkiOiJRWTdqbW5LdHZJcTJSNGl4alBfdGczSnlEZW83U0VkXzJ4eUY2SWxjclVvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjYzMjJmYjU5LTQxOWMtNDg1Mi1iYjZkLWY3MzE5NTllZTc2OSIsImVuY3J5cHRpb25LZXkiOiJRWTdqbW5LdHZJcTJSNGl4alBfdGczSnlEZW83U0VkXzJ4eUY2SWxjclVvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTE0MzU2NyIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI4LCJjb21taXRTaGEiOiJiOWRhYmZjN2M3NTdjZGEyOWE5MDUzODJkOGY3MjdjYzUwM2NhY2RjIiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div>
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit b9dabfc7c757cda29a905382d8f727cc503cacdc. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_REVIEW -->
Cursor Bugbot has reviewed your changes using default effort and found 3 potential issues.
<!-- BUGBOT_FIX_ALL -->
<a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aToyMDAwMzRiMy1mN2ZmLTQ0OWEtOGQ4NC0zMmRhZTA4Y2ZlNTUiLCJlbmNyeXB0aW9uS2V5IjoiSXIzMERvOXNqUldfWVZIYUE0Tk9MNERLUlV2M0RxQkdYQkg5R3pqUHJ3ayIsImJyYW5jaCI6InQyLWNvY2tyb2FjaC0xNDM1NjciLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>
<!-- /BUGBOT_FIX_ALL -->
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN -->
<sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup>
<!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END -->
<sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit b9dabfc7c757cda29a905382d8f727cc503cacdc. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | exact |
| d3 | high | ✓ | exact |
The new tombstone updater builds raw KV batches with no WriteOptions/OriginID. The SQL path tags its writes with OriginIDForLogicalDataReplication: 1, and the KV writer uses originID1Options. These tombstone writes are untagged, so the source rangefeed will not filter them and they replicate back. [grounded: diff] [consensus 3/3: valid=3]
When the delete lost LWW to a live row, the tombstone updater's Deleter still emits secondary-index Dels built from the deleted row's values; only the primary CPut fails. A ConditionFailedError does not poison the txn (txn_coord_sender lets it continue), and the error is swallowed as an LWW loss. Dels already applied on other ranges therefore commit in the shared txn. [grounded: tree pkg/sql/row/deleter.go] [consensus 3/3: valid=3]
This change adds descriptor leases held by the tombstone updaters, which are released only in ReleaseLeases at a checkpoint. On shutdown lrw.close calls Close, which is still a no-op, so leases acquired after the last checkpoint are never released. kvRowProcessor.Close, by contrast, does release its leases. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 3/3: valid=3]
raw comments (5)
**Tombstone update errors are silently dropped**
The outer `if` only enters when the error is an LWW loss. Any other error (lease, deadline, KV failure) falls through to success, so the delete is never retried or sent to the DLQ. Return kvWriteTooOld only when isLwwLoser(err); otherwise return any non-nil err.
```
pkg/crosscluster/logical/tombstone_updater.go:103 }()
pkg/crosscluster/logical/tombstone_updater.go:104 if err != nil && isLwwLoser(err) {
pkg/crosscluster/logical/tombstone_updater.go:105 if isLwwLoser(err) {
pkg/crosscluster/logical/tombstone_updater.go:106 return batchStats{kvWriteTooOld: 1}, nil
pkg/crosscluster/logical/tombstone_updater.go:107 }
pkg/crosscluster/logical/tombstone_updater.go:108 return batchStats{}, err
pkg/crosscluster/logical/tombstone_updater.go:109 }
```
**Lease reuse check is inverted**
`getDeleter` re-acquires the lease while it is still valid and keeps it once expired, so every update churns the lease. After expiry UpdateDeadline fails on every later call. The KV writer reuses only when `Expiration().After(ts)`. Re-acquire only when the descriptor is nil or `!Expiration(ctx).After(timestamp)`.
```
pkg/crosscluster/logical/tombstone_updater.go:158 func (tu *tombstoneUpdater) getDeleter(ctx context.Context, txn *kv.Txn) (row.Deleter, error) {
pkg/crosscluster/logical/tombstone_updater.go:159 timestamp := txn.ProvisionalCommitTimestamp()
pkg/crosscluster/logical/tombstone_updater.go:160 if tu.leased.descriptor == nil || !timestamp.After(tu.leased.descriptor.Expiration(ctx)) {
pkg/crosscluster/logical/tombstone_updater.go:161 tu.ReleaseLeases(ctx)
pkg/crosscluster/logical/lww_kv_processor.go:359 if w.leased != nil {
pkg/crosscluster/logical/lww_kv_processor.go:360 // If the lease is still valid, just use the writer.
pkg/crosscluster/logical/lww_kv_processor.go:361 if w.leased.Expiration(ctx).After(ts) {
pkg/crosscluster/logical/lww_kv_processor.go:362 return w, nil
pkg/crosscluster/logical/lww_kv_processor.go:363 }
pkg/crosscluster/logical/lww_kv_processor.go:364 // The lease is invalid; we'll be getting a new one so release this one.
```
**Tombstone writes replicate back to the source**
The `updateTombstone` batches carry no origin ID, unlike the KV writer's `originID1Options`. The producer only filters out writes tagged with an origin, so in bidirectional LDR this tombstone is streamed back and can ping-pong between clusters. Set `batch.Header.WriteOptions = originID1Options` on both batches.
```
pkg/crosscluster/logical/tombstone_updater.go:89 if err := tu.addToBatch(ctx, txn.KV(), batch, afterRow); err != nil {
pkg/crosscluster/logical/tombstone_updater.go:90 return err
pkg/crosscluster/logical/tombstone_updater.go:91 }
pkg/crosscluster/logical/tombstone_updater.go:92 return txn.KV().Run(ctx, batch)
pkg/crosscluster/logical/lww_kv_processor.go:207 func makeKVBatch(lowPri bool, txn *kv.Txn) *kv.Batch {
pkg/crosscluster/logical/lww_kv_processor.go:208 b := txn.NewBatch()
pkg/crosscluster/logical/lww_kv_processor.go:209 b.Header.WriteOptions = originID1Options
pkg/crosscluster/logical/lww_kv_processor.go:210 if lowPri {
pkg/crosscluster/logical/lww_kv_processor.go:211 b.AdmissionHeader.Priority = int32(admissionpb.BulkLowPri)
pkg/crosscluster/logical/lww_kv_processor.go:212 } else {
```
**Losing delete can remove live secondary index entries**
If the delete lost LWW to a live row, `addToBatch` still Dels secondary index keys built from the deleted event's NULL values. Only the primary CPut fails; Dels on other ranges commit in the shared txn. Write only the primary-index tombstone CPut, or wrap the batch in a savepoint.
```
pkg/crosscluster/logical/tombstone_updater.go:86 // If updateTombstone is called in a transaction, create and run a batch
pkg/crosscluster/logical/tombstone_updater.go:87 // in the transaction.
pkg/crosscluster/logical/tombstone_updater.go:88 batch := txn.KV().NewBatch()
pkg/crosscluster/logical/tombstone_updater.go:89 if err := tu.addToBatch(ctx, txn.KV(), batch, afterRow); err != nil {
pkg/sql/row/deleter.go:147 // Delete the row from any secondary indices.
pkg/sql/row/deleter.go:148 for i, index := range rd.Helper.Indexes {
pkg/sql/row/deleter.go:149 // If the index ID exists in the set of indexes to ignore, do not
pkg/sql/row/deleter.go:150 // attempt to delete from the index.
pkg/sql/row/deleter.go:151 if pm.IgnoreForDel.Contains(int(index.GetID())) {
pkg/sql/row/deleter.go:152 continue
```
**Tombstone updater leases leak when processor closes**
The tombstone updaters now hold descriptor leases, but `sqlRowProcessor.Close` is still a no-op. Leases taken after the last checkpoint are never released when the writer processor shuts down. The kv processor's Close does release its leases. Have sqlRowProcessor.Close call srp.querier.ReleaseLeases(ctx).
```
pkg/crosscluster/logical/lww_row_processor.go:279 func (srp *sqlRowProcessor) ReleaseLeases(ctx context.Context) {
pkg/crosscluster/logical/lww_row_processor.go:280 srp.querier.ReleaseLeases(ctx)
pkg/crosscluster/logical/lww_row_processor.go:281 }
pkg/crosscluster/logical/lww_row_processor.go:283 func (*sqlRowProcessor) Close(ctx context.Context) {}
pkg/crosscluster/logical/lww_kv_processor.go:350 func (p *kvRowProcessor) Close(ctx context.Context) {
pkg/crosscluster/logical/lww_kv_processor.go:351 p.ReleaseLeases(ctx)
pkg/crosscluster/logical/lww_kv_processor.go:352 }
```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | off by one |
| d3 | high | ✓ | exact |
The deleter is built with all secondary indexes and is given values from a CDC delete row, where only PK columns are guaranteed to be set. Deleter.DeleteRow encodes a secondary index key from those values and writes a Del for every index, so the deletes go to keys derived from non-PK values that are absent or NULL. The comment saying the index helpers are 'never really used' does not prevent this. [grounded: tree pkg/crosscluster/logical/tombstone_updater.go] [consensus 3/3: valid=3]
The change makes sqlRowProcessor hold descriptor leases through the tombstone updaters but leaves Close empty. The processor's close() only calls b.Close, so leases acquired after the last checkpoint are never released. kvRowProcessor.Close, by contrast, calls ReleaseLeases. This is an incomplete change. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 3/3: valid=3]
The comment is stale now that the method delegates to the querier, but this is only a documentation issue, not a behaviour bug. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 3/3: minor=3]
raw comments (5)
**Tombstone update errors are silently swallowed**
In `updateTombstone`, the outer `if err != nil && isLwwLoser(err)` means any error that isn't an LWW loss falls through to `return batchStats{}, nil`. The inner `return batchStats{}, err` is unreachable. Use `if err != nil { if isLwwLoser(err) { return batchStats{kvWriteTooOld: 1}, nil }; return batchStats{}, err }`.
```
pkg/crosscluster/logical/tombstone_updater.go:87 // in the transaction.
pkg/crosscluster/logical/tombstone_updater.go:88 batch := txn.KV().NewBatch()
pkg/crosscluster/logical/tombstone_updater.go:89 if err := tu.addToBatch(ctx, txn.KV(), batch, afterRow); err != nil {
pkg/crosscluster/logical/tombstone_updater.go:90 return err
```
**Lease validity check in getDeleter is inverted**
`getDeleter` re-acquires the lease when `!timestamp.After(expiration)`, which is when the lease is still valid. The comparison is the wrong way round. A lease that has expired (timestamp after expiration) is reused. Re-acquire only when the descriptor is nil or `!expiration.After(timestamp)`.
```
pkg/crosscluster/logical/tombstone_updater.go:159 timestamp := txn.ProvisionalCommitTimestamp()
pkg/crosscluster/logical/tombstone_updater.go:160 if tu.leased.descriptor == nil || !timestamp.After(tu.leased.descriptor.Expiration(ctx)) {
pkg/crosscluster/logical/tombstone_updater.go:161 tu.ReleaseLeases(ctx)
pkg/crosscluster/logical/lww_kv_processor.go:359 if w.leased != nil {
pkg/crosscluster/logical/lww_kv_processor.go:360 // If the lease is still valid, just use the writer.
pkg/crosscluster/logical/lww_kv_processor.go:361 if w.leased.Expiration(ctx).After(ts) {
pkg/crosscluster/logical/lww_kv_processor.go:362 return w, nil
pkg/crosscluster/logical/lww_kv_processor.go:363 }
pkg/crosscluster/logical/lww_kv_processor.go:364 // The lease is invalid; we'll be getting a new one so release this one.
pkg/crosscluster/logical/lww_kv_processor.go:365 w.leased.Release(ctx)
```
**Tombstone update writes bogus secondary index deletes**
`addToBatch` builds the delete values from the cdc delete row, where only primary-key columns are guaranteed to be set. `Deleter.DeleteRow` then derives secondary index keys from those values and writes deletes at bogus keys. Pass a Deleter with no secondary indexes, since only the primary tombstone needs its timestamp updated.
```
pkg/crosscluster/logical/tombstone_updater.go:119 }
pkg/crosscluster/logical/tombstone_updater.go:121 tu.scratch = tu.scratch[:0]
pkg/crosscluster/logical/tombstone_updater.go:123 // Note that the columns in the cdcevent row are decoded using a descriptor
pkg/crosscluster/logical/tombstone_updater.go:124 // for the source column, whereas the row.Deleter column list is initialized
pkg/ccl/changefeedccl/cdcevent/event.go:65 // deleted is true if row is a deletion. In this case, only the primary
pkg/ccl/changefeedccl/cdcevent/event.go:66 // key columns are guaranteed to be set in `datums`.
pkg/ccl/changefeedccl/cdcevent/event.go:67 deleted bool
```
**Tombstone leases leaked when processor closes**
`sqlRowProcessor.Close` is still an empty function. The tombstone updaters now hold descriptor leases, and these are released only by `ReleaseLeases` at checkpoints. Leases taken after the last checkpoint are never released on shutdown. `kvRowProcessor.Close` calls `ReleaseLeases`. Have `sqlRowProcessor.Close` call `srp.ReleaseLeases(ctx)`.
```
pkg/crosscluster/logical/lww_row_processor.go:283 func (*sqlRowProcessor) Close(ctx context.Context) {}
pkg/crosscluster/logical/tombstone_updater.go:158 func (tu *tombstoneUpdater) getDeleter(ctx context.Context, txn *kv.Txn) (row.Deleter, error) {
pkg/crosscluster/logical/tombstone_updater.go:159 timestamp := txn.ProvisionalCommitTimestamp()
pkg/crosscluster/logical/tombstone_updater.go:160 if tu.leased.descriptor == nil || !timestamp.After(tu.leased.descriptor.Expiration(ctx)) {
pkg/crosscluster/logical/lww_kv_processor.go:350 func (p *kvRowProcessor) Close(ctx context.Context) {
pkg/crosscluster/logical/lww_kv_processor.go:351 p.ReleaseLeases(ctx)
pkg/crosscluster/logical/lww_kv_processor.go:352 }
```
**Stale comment says ReleaseLeases is a no-op**
The doc comment on `sqlRowProcessor.ReleaseLeases` still says it is a no-op because each query releases its own leases. It now delegates to the querier and releases the tombstone updaters' leases. Update the comment to say it releases the tombstone updaters' leases.
```
pkg/crosscluster/logical/lww_row_processor.go:277 // ReleaseLeases implements the BatchHandler interface but is a no-op since each
pkg/crosscluster/logical/lww_row_processor.go:278 // query does this itself.
pkg/crosscluster/logical/lww_row_processor.go:279 func (srp *sqlRowProcessor) ReleaseLeases(ctx context.Context) {
pkg/crosscluster/logical/lww_row_processor.go:280 srp.querier.ReleaseLeases(ctx)
```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | exact |
| d3 | high | ✓ | exact |
raw comments (3)
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Tombstone errors are swallowed**
When tombstone preparation, lease acquisition, deadline setup, or KV execution returns a non-LWW error, this condition skips the error-return branch and reports success. Replication can therefore advance without updating the tombstone timestamp, allowing a later, older update to resurrect the deleted row.
```suggestion
if err != nil {
if isLwwLoser(err) {
return batchStats{kvWriteTooOld: 1}, nil
}
return batchStats{}, err
}
return batchStats{}, nil
```
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Lease validity check is reversed**
This predicate reacquires the descriptor while the transaction timestamp is at or before expiration, but retains it after expiration. A later tombstone update can therefore use an obsolete transaction deadline and fail instead of acquiring a valid descriptor; with the current error handling, that failed update can be acknowledged without advancing the tombstone.
```suggestion
if tu.leased.descriptor == nil || !timestamp.Less(tu.leased.descriptor.Expiration(ctx)) {
```
<h3>Greptile Summary</h3>
This PR adds a KV-level tombstone updater for logical-replication deletes that affect no live SQL row, wires descriptor leases into the LWW processor, and adds a three-database regression test.
- Updates tombstone origin timestamps so newer replicated deletes continue to win LWW conflicts.
- Caches destination descriptors and row deleters per replicated table.
- Adds lease cleanup through the row-processor lifecycle.
- Contains error-propagation and lease-validity defects in the new updater.
<h3>Confidence Score: 3/5</h3>
The PR is not safe to merge because tombstone updates can silently fail and expired descriptor leases are retained.
Non-LWW updater errors are converted to success, which can leave stale tombstones and permit deleted rows to reappear; the reversed lease check independently makes such failures likely after a cached lease expires.
**Files Needing Attention:** pkg/crosscluster/logical/tombstone_updater.go
<details><summary><h3>Important Files Changed</h3></summary>
| Filename | Overview |
|----------|----------|
| pkg/crosscluster/logical/tombstone_updater.go | Adds descriptor-backed tombstone writes, but suppresses non-LWW failures and reverses the cached-lease validity check. |
| pkg/crosscluster/logical/lww_row_processor.go | Routes zero-row replicated deletes through the tombstone updater and adds lease cleanup to the querier interface. |
| pkg/crosscluster/logical/logical_replication_writer_processor.go | Supplies the SQL codec and concrete lease manager required by the new updater. |
| pkg/crosscluster/logical/logical_replication_job_test.go | Adds an end-to-end regression scenario proving that a replicated delete must advance an existing tombstone timestamp. |
</details>
<details open><summary><h3>Flowchart</h3></summary>
```mermaid
%%{init: {'theme': 'neutral'}}%%
flowchart TD
A[Replicated DELETE] --> B[Execute conditional SQL DELETE]
B -->|one row deleted| C[Success]
B -->|zero rows| D[Look up per-table tombstone updater]
D --> E[Acquire or reuse destination descriptor lease]
E --> F[Build origin-timestamp conditional tombstone write]
F --> G{KV result}
G -->|LWW loser| H[Count write-too-old and succeed]
G -->|success| C
G -->|other error| I[Must propagate error]
```
</details>
<!-- greptile_other_comments_section -->
<sub>Reviews (1): Last reviewed commit: ["Update pkg/crosscluster/logical/BUILD.ba..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/5b81f019f761186c4bf9b2ee3d7b9ebd40a7e36c) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=64284314)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | exact |
| d3 | high | ✓ | exact |
The new tombstone batches are built with plain NewBatch() and never set WriteOptions.OriginID. The KV processor sets it explicitly via makeKVBatch (originID1Options), and the row package does not set OriginID either. So the tombstone write looks local. [grounded: tree pkg/crosscluster/logical/tombstone_updater.go] [consensus 3/3: valid=3]
The change makes sqlRowProcessor hold leases through tombstoneUpdater, but Close stays a no-op. The writer's close path only calls b.Close, so leases taken after the last checkpoint are never released. kvRowProcessor.Close calls ReleaseLeases. [grounded: tree pkg/crosscluster/logical/lww_row_processor.go] [consensus 3/3: valid=3]
The outer condition already requires isLwwLoser, so the inner return err can never run. Every other error falls through to return batchStats{}, nil and is swallowed. [grounded: tree pkg/crosscluster/logical/tombstone_updater.go] [consensus 3/3: valid=3]
Same root cause as finding 2. The batch has no OriginID, and the rangefeed only omits writes with OriginID != 0, so the delete is echoed back in bidirectional setups. [grounded: tree pkg/crosscluster/logical/tombstone_updater.go] [consensus 3/3: valid=3]
The lease is reacquired when the timestamp is at or before expiration, which is when the lease is still valid. Once it has expired, the lease is kept. This is the reverse of the kvRowProcessor check, Expiration(ctx).After(ts). [grounded: tree pkg/crosscluster/logical/tombstone_updater.go] [consensus 3/3: valid=3]
raw comments (7)
[P1] Non-LWW errors from the tombstone update are silently swallowed
The outer check is `err != nil && isLwwLoser(err)`. Any other error skips the block and reaches `return batchStats{}, nil`, so the inner `return batchStats{}, err` can never run. These errors include lease acquisition failures, `writeableColunms` errors, `DatumsNamed` errors, KV/RPC failures and retryable txn errors. In each case the tombstone's origin timestamp is not updated, but `DeleteRow` still reports success. The event is then neither retried nor sent to purgatory, which brings back the LWW divergence this change is meant to fix. In the txn path, the caller also continues with a KV txn that may already have failed.
[P1] Lease validity check is inverted in getDeleter
The code re-acquires when `!timestamp.After(expiration)`, which means whenever the cached lease is still valid. When the lease has expired (`timestamp` after `Expiration`), it keeps the stale descriptor and deleter. `kvRowProcessor.getWriter` uses the opposite test: reuse the lease when `Expiration(ctx).After(ts)`. As written, every tombstone update releases and re-acquires the lease. Once the lease expires, the processor keeps using the old descriptor and calls `txn.UpdateDeadline` with a deadline already in the past, so those tombstone updates fail. Combined with the error swallowing above, they fail silently.
[P2] Tombstone write is not tagged with the LDR origin ID
The SQL apply path sets `OriginIDForLogicalDataReplication: 1` (lww_row_processor.go:401), and the KV processor sets `b.Header.WriteOptions = originID1Options`. LDR rangefeeds rely on this to filter out writes that came from replication. The batches built here through `txn.KV().NewBatch()` / `txn.NewBatch()` never set `WriteOptions.OriginID`. In a bidirectional setup, any tombstone rewritten by this path shows up as a local write on the destination's outgoing stream. The delete is then replicated back to the source instead of being filtered out.
[P2] sqlRowProcessor.Close does not release the new tombstone-updater leases
This change makes `sqlRowProcessor` hold descriptor leases through `tombstoneUpdater`, but `Close` is still a no-op. Leases are only released at checkpoints (logical_replication_writer_processor.go:666). The writer processor's close path (line 485) only calls `b.Close`. So any lease acquired after the last checkpoint stays referenced after the processor shuts down, for example when the job is paused, fails or is replanned. A leaked lease reference can block schema changes on the destination table while they wait for the old descriptor version to drain. `kvRowProcessor.Close` avoids this by calling `p.ReleaseLeases(ctx)`.
[P1] updateTombstone discards every error that isn't an LWW loss
The outer check is `err != nil && isLwwLoser(err)`, so the inner `return batchStats{}, err` can never run. Any other error falls through to `return batchStats{}, nil`. That covers lease acquisition failures, `UpdateDeadline` failures, `DatumsNamed` assertion errors, KV errors and retryable txn errors. The replicated delete is then reported as applied even though the tombstone was never written, so the row is never retried or sent to the DLQ. In the multi-row batch path, a retryable error is also hidden from the enclosing `isql.Txn`. The outer condition should be `err != nil`.
[P1] Tombstone write has no LDR origin ID, so bidirectional streams echo it back
Both batches are created with a plain `NewBatch()`, so `Header.WriteOptions` is never set. The SQL path sets the origin ID through the `OriginIDForLogicalDataReplication` session override, and the KV path uses `makeKVBatch` with `originID1Options`; this path does neither. The tombstone is therefore written with `OriginID == 0` and looks like a local write. The producer rangefeed only omits remote writes when `OriginID != 0` (`catchup_scan.go:347`), so in a bidirectional LDR setup this replicated delete is emitted back to the source as a new local delete with a newer MVCC timestamp. The source then applies the same tombstone update without an origin ID and sends it back again, which can turn into a ping-pong of deletes. The batch should set `WriteOptions = originID1Options`, as `makeKVBatch` does.
[P2] Lease-expiry check is reversed
The lease is reacquired when `!timestamp.After(expiration)`, meaning while it is still valid. Once the txn timestamp passes the expiration, the stale lease is kept. As a result, every tombstone update releases and reacquires a lease that is still good. After a lease expires but before the next checkpoint calls `ReleaseLeases`, the code keeps using the expired descriptor and passes its expiration to `txn.UpdateDeadline`. That returns an error when the deadline is below the read timestamp, and the error is then silently discarded by the bug above. The KV processor uses the correct form: keep the lease only if `Expiration(ctx).After(ts)`.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✓ | off by one |
| d3 | high | ✓ | exact |
The outer condition only enters for LWW losers, so all other errors fall through to `return batchStats{}, nil`. The failure is swallowed and the row counts as applied. [grounded: diff] [consensus 3/3: valid=3]
This reacquires the lease while it is still valid (timestamp <= expiration) and keeps an expired one. That is the opposite of kvRowProcessor.getWriter, which reuses a lease only while `Expiration(ctx).After(ts)`. [grounded: diff] [consensus 3/3: valid=3]
The tombstone batch is created with a plain NewBatch and never gets `WriteOptions = originID1Options`, unlike makeKVBatch in the KV processor. OriginTimestampCPutHelper does not set an origin ID either, so the write is not tagged as replicated. [grounded: diff] [consensus 3/3: valid=3]
raw comments (5)
[P1] Non-LWW errors from updateTombstone are silently swallowed
The condition is `if err != nil && isLwwLoser(err) { ... }`, so the inner `return batchStats{}, err` is unreachable. Any error that isn't an LWW-loser (lease acquisition failure, `UpdateDeadline` failure, batch/commit error, retryable error from the 1PC txn, or a `DatumsNamed` assertion failure) falls through to `return batchStats{}, nil`. DeleteRow then reports success even though the tombstone was never updated. This silently reintroduces the stale-tombstone LWW bug that this change is meant to fix, and it also hides retryable errors from the caller. The check should be `if err != nil { if isLwwLoser(err) { ... } return err }`.
[P2] Lease refresh condition in getDeleter is inverted
`!timestamp.After(expiration)` is true while the lease is still valid (ts ‹= expiration). The code therefore releases and re-acquires the lease, and rebuilds the deleter and column list, on every call while the lease is valid. It does not refresh once the timestamp is past the expiration. `kvRowProcessor.getWriter` reuses the writer when `Expiration(ctx).After(ts)` and re-acquires otherwise. In this code, a provisional commit timestamp later than the cached expiration reuses the expired descriptor. `UpdateDeadline` then fails with a deadline-exceeded error, which is currently swallowed (see the previous finding). The condition should re-acquire only when `!expiration.After(timestamp)`.
[P1] Non-LWW errors from the tombstone write are swallowed
The condition `err != nil && isLwwLoser(err)` only enters the block for LWW losers, so every other error (lease acquisition failure, txn retry/abort, unique/condition failure, encoding errors) falls through to `return batchStats{}, nil`. The inner `if isLwwLoser(err)` is redundant and the `return batchStats{}, err` is unreachable. When the delete reports 0 rows and the tombstone write fails for any other reason, the row is reported as successfully applied, so it is neither retried nor sent to the DLQ. In the batched-txn path the error is also hidden from the caller. The fix is to return the error when it is not an LWW loser
[P1] Lease reacquire condition is inverted
The check `!timestamp.After(expiration)` is true while the lease is still valid, so a valid lease is released and reacquired on every call. When the lease has expired (`timestamp.After(expiration)`), the check is false and the expired descriptor is reused. `txn.UpdateDeadline(ctx, expiredExpiration)` then fails, or the write uses a stale descriptor. This path runs on every delete that matches no row, so it causes lease churn on the hot path and errors once leases expire. `kvRowProcessor.getWriter` reuses the lease only while `Expiration(ctx).After(ts)`; this check needs the opposite polarity
[P2] Tombstone CPut does not set the LDR origin ID, so it can loop back
The tombstone update is written through a plain `txn.NewBatch()` or `tx.NewBatch()` without `Header.WriteOptions = originID1Options`. The SQL delete path gets its origin ID from `ieOverrideBase` (OriginIDForLogicalDataReplication), and `kvRowProcessor` sets `originID1Options` via `makeKVBatch`. In a bidirectional LDR setup, the tombstone rewrite on the destination is therefore not tagged as replicated. It is emitted on the reverse stream and applied back to the source, which defeats the data-looping prevention
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
| Defect | Severity | Found | Grade |
|---|---|---|---|
| d2 | high | ✗ | — |
| d3 | high | ✗ | — |
raw comments (2)
<h3>Qodo is busy working</h3>
Check back in a few minutes. Qodo's code review agents are on it.
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<img src="https://www.qodo.ai/wp-content/uploads/2026/01/anteater-looking-at-ants-01-transparent-bg-croped-ezgif.com-optimize-1.gif" width="30%">
<h3>PR Summary by Qodo</h3>
Preserve LWW tombstone timestamps during logical replication
<code>🐞 Bug fix</code> <code>🧪 Tests</code> <code>🕐 20-40 Minutes</code>
<img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider">
<details>
<summary>AI Description</summary>
<dl>
<dd>
<br/>
><pre>
>• Refreshes tombstone origin timestamps when replicated deletes encounter already-deleted rows.
>• Manages descriptor leases for direct KV tombstone conditional writes.
>• Adds a three-database regression test proving newer deletes beat older updates.
></pre>
</dd>
</dl>
</details>
<details>
<summary>Diagram</summary>
<dl>
<dd>
<br/>
```mermaid
sequenceDiagram
participant Source as Source Stream
participant Writer as Replication Writer
participant LWW as LWW Querier
participant Updater as Tombstone Updater
participant Lease as Lease Manager
participant KV as KV Store
Source->>Writer: Replicated delete
Writer->>LWW: Apply delete
LWW->>LWW: Check affected rows
alt Existing row deleted
LWW-->>Writer: Delete applied
else Tombstone or LWW loss
LWW->>Updater: Refresh tombstone
Updater->>Lease: Acquire descriptor
Lease-->>Updater: Leased descriptor
Updater->>KV: Conditional tombstone write
KV-->>Updater: Applied or older write
Updater-->>Writer: Conflict result
end
```
</dd>
</dl>
</details>
<details>
<summary>High-Level Assessment</summary>
<dl>
<dd>
<br/>
>The following are alternative approaches to this PR:
<details>
<summary><b>1. Route all deletes through the KV processor</b></summary>
<dl>
<dd>
- ➕ Uses an existing KV-based LWW write path
- ➕ Avoids a SQL-delete-then-fallback sequence
- ➖ Broadens behavior changes beyond the tombstone bug
- ➖ Risks bypassing validated-mode SQL semantics and constraint handling
- ➖ Requires adapting processor state and previous-value assumptions
</dd>
</dl>
</details>
<details>
<summary><b>2. Add SQL-level tombstone update support</b></summary>
<dl>
<dd>
- ➕ Keeps all validated replication writes behind SQL execution
- ➕ Could provide reusable tombstone semantics for other callers
- ➖ Requires a substantially broader SQL or internal-executor feature
- ➖ Increases implementation and compatibility risk for a targeted regression
</dd>
</dl>
</details>
>**Recommendation:** Keep the targeted SQL fallback implemented by this PR. It preserves existing validated-mode behavior for live rows while using direct KV operations only when SQL cannot modify an existing tombstone; reusing the full KV processor or extending SQL would create substantially broader changes.
</dd>
</dl>
</details>
<details>
<summary> Files changed (7) <code> +337 / -15 </code> </summary>
<dl>
<dd>
<br/>
<details>
<summary>Bug fix (3) <code> +234 / -4 </code></summary>
<dl>
<dd>
<details>
<summary>logical_replication_writer_processor.go<code>Provide codec and lease manager to SQL handlers</code> <code>+3/-0</code></summary>
<br/>
>Provide codec and lease manager to SQL handlers
>
><pre>
>• Passes the flow SQL codec and concrete lease manager into SQL row processor construction so tombstone fallback writes can encode destination keys safely.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-f6e24d38db7bfcad57c57d8f8b95331d9cbf8ca8a5084f743e89545d3de2d157'>pkg/crosscluster/logical/logical_replication_writer_processor.go</a>
</details>
<details>
<summary>lww_row_processor.go<code>Fallback to KV tombstone updates for zero-row deletes</code> <code>+47/-4</code></summary>
<br/>
>Fallback to KV tombstone updates for zero-row deletes
>
><pre>
>• Detects replicated SQL deletes that affect no live row and invokes a per-table tombstone updater. Extends queriers with lease cleanup and propagates codec, database, and lease-manager dependencies through processor construction.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-da970f90e59957fb8e63879bc37526f059394196015371b2c64ac53f6424dd2d'>pkg/crosscluster/logical/lww_row_processor.go</a>
</details>
<details>
<summary>tombstone_updater.go<code>Implement lease-backed MVCC tombstone timestamp updates</code> <code>+184/-0</code></summary>
<br/>
>Implement lease-backed MVCC tombstone timestamp updates
>
><pre>
>• Introduces a per-table helper that acquires destination descriptors, decodes source datums, and issues conditional row deletes with the replicated origin timestamp. It supports existing or standalone transactions, enforces lease deadlines, reuses buffers, and treats older LWW writes as successful no-ops.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-f41512aa20a09e74e1a7acc3c698745c73ca9cda4d452ed0bde66d5a43e3869a'>pkg/crosscluster/logical/tombstone_updater.go</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Refactor (1) <code> +4 / -0 </code></summary>
<dl>
<dd>
<details>
<summary>udf_row_processor.go<code>Implement querier lease-release contract</code> <code>+4/-0</code></summary>
<br/>
>Implement querier lease-release contract
>
><pre>
>• Adds a no-op lease release method to the UDF applier querier because it does not retain descriptor leases.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-91c1b85f3f1745ec5cb3d63a076c6f15abcabe3fa828e9267fbf8ad8bca51eb4'>pkg/crosscluster/logical/udf_row_processor.go</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Tests (2) <code> +97 / -11 </code></summary>
<dl>
<dd>
<details>
<summary>logical_replication_job_test.go<code>Cover three-way tombstone conflict regression</code> <code>+80/-0</code></summary>
<br/>
>Cover three-way tombstone conflict regression
>
><pre>
>• Adds an end-to-end test with two sources and one destination. It verifies that a newer replicated delete refreshes an existing tombstone and defeats an older replicated update under LWW resolution.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-14721cc326ec46270739882bd5237c7a80ff8f040d02130cc4a752be46d09945'>pkg/crosscluster/logical/logical_replication_job_test.go</a>
</details>
<details>
<summary>lww_row_processor_test.go<code>Supply lease dependencies and release test leases</code> <code>+17/-11</code></summary>
<br/>
>Supply lease dependencies and release test leases
>
><pre>
>• Updates SQL processor test and benchmark construction for the new codec and lease-manager arguments. Test setup now returns cleanup callbacks that release descriptor leases.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-2c5740d1aa8c7f62a1c518362cb8166dcc1702e7ebbecc5687d6efd51a9c5ab2'>pkg/crosscluster/logical/lww_row_processor_test.go</a>
</details>
</dd>
</dl>
</details>
<details>
<summary>Other (1) <code> +2 / -0 </code></summary>
<dl>
<dd>
<details>
<summary>BUILD.bazel<code>Register tombstone updater and lease dependencies</code> <code>+2/-0</code></summary>
<br/>
>Register tombstone updater and lease dependencies
>
><pre>
>• Adds the tombstone updater to the logical replication library and exposes the catalog lease dependency to tests.
></pre>
>
><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/27/files#diff-956fab78ca74cbde04e82530a01bc83fbec99700598d0ac35ab93083915565b3'>pkg/crosscluster/logical/BUILD.bazel</a>
</details>
</dd>
</dl>
</details>
</dd>
</dl>
</details>