crbench//

← all cases

Crud writer feeds tombstone updates a row without stored computed columns, panicking the Deleter

Tier 2 hard go · cross-file · real merged PR from cockroachdb/cockroach #143988

Ground-truth defects

d1 · correctness · high · pkg/crosscluster/logical/event_decoder.go:79
The new eventDecoder builds each event's row from `getPhysicalColumnsSchema`, which drops every computed column, stored ones included. When the crud writer applies a replicated delete for a row the destination does not have, the table handler passes that row to `tombstoneUpdater.updateTombstone`. The tombstone updater's row.Deleter is built from `writeableColunms`, which keeps stored computed columns, and indexes the row by position. On a table with a stored computed column, every later column's datum is shifted. For example, on `(k INT PRIMARY KEY, c INT AS (k+1) STORED, v INT, INDEX (v))` the Deleter reads `v` at ordinal 2 of a 2-element row, and secondary-index encoding panics with index out of range. A stored computed column ahead of a primary-key column shifts the primary-key datum in the same way.

The change under review

diff (truncated — full diff)
diff --git a/pkg/crosscluster/logical/BUILD.bazel b/pkg/crosscluster/logical/BUILD.bazel
index 31d34a111d4..6045e1cbf0d 100644
--- a/pkg/crosscluster/logical/BUILD.bazel
+++ b/pkg/crosscluster/logical/BUILD.bazel
@@ -17,6 +17,7 @@ go_library(
         "purgatory.go",
         "range_stats.go",
         "replication_statements.go",
+        "sql_crud_writer.go",
         "sql_row_reader.go",
         "sql_row_writer.go",
         "table_batch_handler.go",
diff --git a/pkg/crosscluster/logical/event_decoder.go b/pkg/crosscluster/logical/event_decoder.go
index 375452ee4ef..17496beb3a5 100644
--- a/pkg/crosscluster/logical/event_decoder.go
+++ b/pkg/crosscluster/logical/event_decoder.go
@@ -6,11 +6,35 @@
 package logical
 
 import (
+	"context"
+
+	"github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdcevent"
+	"github.com/cockroachdb/cockroach/pkg/repstream/streampb"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descs"
 	"github.com/cockroachdb/cockroach/pkg/sql/sem/tree"
+	"github.com/cockroachdb/cockroach/pkg/sql/types"
 	"github.com/cockroachdb/cockroach/pkg/util/hlc"
+	"github.com/cockroachdb/errors"
 )
 
+// eventDecoder takes a KV from the source cluster and decodes it into datums
+// that are appropriate for the destination table.
+type eventDecoder struct {
+	decoder   cdcevent.Decoder
+	srcToDest map[descpb.ID]destinationTable
+
+	// TODO(jeffswenson): clean this interface up. There's a problem with
+	// layering that requires the event decoder to know about the most recent
+	// row. If a batch fails to process, its broken up into batches of size 1 and
+	// those are retried until they succeed or are DLQ'd. The batch handler is
+	// responsible for decoding rows, but the distsql processor is responsible
+	// for calling the batch handler and adding rows to the DLQ.
+	lastRow cdcevent.Row
+}
+
 // decodedEvent is constructed from a replication stream event. The replication
 // stream event is encoded using the source table descriptor. The decoded must
 // contain datums that are compatible with the destination table descriptor.
@@ -37,3 +61,124 @@ type decodedEvent struct {
 	// prevRow may still lose LWW if there is a recent tombstone.
 	prevRow tree.Datums
 }
+
+func newEventDecoder(
+	ctx context.Context,
+	descriptors descs.DB,
+	settings *cluster.Settings,
+	procConfigByDestID map[descpb.ID]sqlProcessorTableConfig,
+) (*eventDecoder, error) {
+	srcToDest := make(map[descpb.ID]destinationTable, len(procConfigByDestID))
+	err := descriptors.DescsTxn(ctx, func(ctx context.Context, txn descs.Txn) error {
+		for dstID, s := range procConfigByDestID {
+			descriptor, err := txn.Descriptors().GetLeasedImmutableTableByID(ctx, txn.KV(), dstID)
+			if err != nil {
+				return err
+			}
+
+			columns := getPhysicalColumnsSchema(descriptor)
+			columnNames := make([]string, 0, len(columns))
+			for _, column := range columns {
+				columnNames = append(columnNames, column.column.GetName())
+			}
+
+			srcToDest[s.srcDesc.GetID()] = destinationTable{
+				id:      dstID,
+				columns: columnNames,
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	decoder, err := newCdcEventDecoder(ctx, procConfigByDestID, settings)
+	if err != nil {
+		return nil, err
+	}
+
+	return &eventDecoder{
+		decoder:   decoder,
+		srcToDest: srcToDest,
+	}, nil
+}
+
+func (d *eventDecoder) decodeEvent(
+	ctx context.Context, event streampb.StreamEvent_KV,
+) (decodedEvent, error) {
+	decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false)
+	if err != nil {
+		return decodedEvent{}, err
+	}
+
+	dstTable, ok := d.srcToDest[decodedRow.TableID]
+	if !ok {
+		return decodedEvent{}, errors.AssertionFailedf("table %d not found", decodedRow.TableID)
+	}
+
+	row := make(tree.Datums, 0, len(dstTable.columns))
+	row, err = appendDatums(row, decodedRow, dstTable.columns)
+	if err != nil {
+		return decodedEvent{}, err
+	}
+	d.lastRow = decodedRow
+
+	var prevKV roachpb.KeyValue
+	prevKV.Key = event.KeyValue.Key
+	prevKV.Value = event.PrevValue
+
+	decodedPrevRow, err := d.decoder.DecodeKV(ctx, prevKV, cdcevent.PrevRow, event.PrevValue.Timestamp, false)
+	if err != nil {
+		return decodedEvent{}, err
+	}
+
+	prevRow := make(tree.Datums, 0, len(dstTable.columns))
+	prevRow, err = appendDatums(prevRow, decodedPrevRow, dstTable.columns)
+	if err != nil {
+		return decodedEvent{}, err
+	}
+
+	return decodedEvent{
+		dstDescID:       dstTable.id,
+		isDelete:        decodedRow.IsDeleted(),
+		originTimestamp: event.KeyValue.Value.Timestamp,
+		row:             row,
+		prevRow:         prevRow,
+	}, nil
+}
+
+// appendDatums appends datums for the specified column names from the cdcevent.Row
+// to the datums slice and returns the updated slice.
+func appendDatums(datums tree.Datums, row cdcevent.Row, columnNames []string) (tree.Datums, error) {
+	it, err := row.DatumsNamed(columnNames)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := it.Datum(func(d tree.Datum, col cdcevent.ResultColumn) error {
+		if dEnum, ok := d.(*tree.DEnum); ok {
+			// Override the type to Unknown to avoid a mismatched type OID error
+			// during execution. Note that Unknown is the type used by default
+			// when a SQL statement is executed without type hints.
+			//
+			// TODO(jeffswenson): this feels like the wrong place to do this,
+			// but its inspired by the implementation in queryBuilder.AddRow.
+			//
+			// Really we should be mapping from the source datum type to the
+			// destination datum type.
+			dEnum.EnumTyp = types.Unknown
+		}
+		datums = append(datums, d)
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	return datums, nil
+}
+
+type destinationTable struct {
+	id      descpb.ID
+	columns []string
+}
diff --git a/pkg/crosscluster/logical/logical_replication_job_test.go b/pkg/crosscluster/logical/logical_replication_job_test.go
index 3c2bd3f5431..9fb23dbafe6 100644
--- a/pkg/crosscluster/logical/logical_replication_job_test.go
+++ b/pkg/crosscluster/logical/logical_replication_job_test.go
@@ -1403,13 +1403,13 @@ func TestTombstoneUpdate(t *testing.T) {
 
 	// 5. Replicate the delete from 'src-a' -> 'dst'
 	var jobIDSrcA jobspb.JobID
-	dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH MODE = VALIDATED, CURSOR = $2",
+	dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH CURSOR = $2",
 		urlSrcA, start.AsOfSystemTime()).Scan(&jobIDSrcA)
 	WaitUntilReplicatedTime(t, s.Clock().Now(), dst, jobIDSrcA)
 
 	// 6. Replicate the update from 'src-b' -> 'dst'
 	var jobIDSrcB jobspb.JobID
-	dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH MODE = VALIDATED, CURSOR = $2",
+	dst.QueryRow(t, "CREATE LOGICAL REPLICATION STREAM FROM TABLE tab ON $1 INTO TABLE tab WITH CURSOR = $2",
 		urlSrcB, start.AsOfSystemTime()).Scan(&jobIDSrcB)
 	WaitUntilReplicatedTime(t, s.Clock().Now(), dst, jobIDSrcB)
 
diff --git a/pkg/crosscluster/logical/logical_replication_writer_processor.go b/pkg/crosscluster/logical/logical_replication_writer_processor.go
index 60a3b542657..c12e64d7c3a 100644
--- a/pkg/crosscluster/logical/logical_replication_writer_processor.go
+++ b/pkg/crosscluster/logical/logical_replication_writer_processor.go
@@ -104,16 +104,20 @@ const (
 	// is deprecated because it does not support the full set of features of the
 	// SQL writer.
 	writerTypeLegacyKV writerType = "legacy-kv"
+
+	// writerTypeCRUD is the shiny new sql writer that uses explicit reads,
+	// inserts, updates, and deletes instead of upserts.
+	writerTypeCRUD writerType = "crud"
 )
 
 var immediateModeWriter = settings.RegisterStringSetting(
 	settings.ApplicationLevel,
 	"logical_replication.consumer.immediate_mode_writer",
 	"the writer to use when in immediate mode",
-	metamorphic.ConstantWithTestChoice("logical_replication.consumer.immediate_mode_writer", string(writerTypeSQL), string(writerTypeLegacyKV)),
+	metamorphic.ConstantWithTestChoice("logical_replication.consumer.immediate_mode_writer", string(writerTypeSQL), string(writerTypeLegacyKV), string(writerTypeCRUD)),
 	settings.WithValidateString(func(sv *settings.Values, val string) error {
-		if val != string(writerTypeSQL) && val != string(writerTypeLegacyKV) {
-			return errors.Newf("immediate mode writer must be either 'sql' or 'legacy-kv', got '%s'", val)
+		if val != string(writerTypeSQL) && val != string(writerTypeLegacyKV) && val != string(writerTypeCRUD) {
+			return errors.Newf("immediate mode writer must be either 'sql', 'legacy-kv', or 'crud', got '%s'", val)
 		}
 		return nil
 	}),
@@ -475,7 +479,11 @@ func (lrw *logicalReplicationWriterProcessor) close() {
 	}
 
 	for _, b := range lrw.bh {
-		b.Close(lrw.Ctx())
+		// The batch handler may be nil if the context was cancelled during
+		// initialization.
+		if b != nil {
+			b.Close(lrw.Ctx())
+		}
 	}
 
 	// Update the global retry queue gauges to reflect that this queue is going
@@ -750,6 +758,11 @@ func (lrw *logicalReplicationWriterProcessor) setupBatchHandlers(ctx context.Con
 			if err != nil {
 				return err
 			}
+		case writerTypeCRUD:
+			rp, err = newCrudSqlWriter(ctx, flowCtx.Cfg, flowCtx.EvalCtx, sd, lrw.spec.Discard, lrw.configByTable, jobspb.JobID(lrw.spec.JobID))
+			if err != nil {
+				return err
+			}
 		default:
 			return errors.AssertionFailedf("unknown logical replication writer type: %s", lrw.spec.WriterType)
 		}
@@ -1120,7 +1133,6 @@ func (lrw *logicalReplicationWriterProcessor) shouldRetryLater(
 		return tooOld
 	}
 
-	// TODO(dt): maybe this should only be constraint violation errors?
 	return retryAllowed
 }
 
diff --git a/pkg/crosscluster/logical/lww_kv_processor.go b/pkg/crosscluster/logical/lww_kv_processor.go
index e37906723fc..c023b601e0a 100644
--- a/pkg/crosscluster/logical/lww_kv_processor.go
+++ b/pkg/crosscluster/logical/lww_kv_processor.go
@@ -16,6 +16,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
 	"github.com/cockroachdb/cockroach/pkg/repstream/streampb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
 	"github.com/cockroachdb/cockroach/pkg/sql/catalog/lease"
@@ -60,12 +61,37 @@ func newKVRowProcessor(
 	spec execinfrapb.LogicalReplicationWriterSpec,
 	procConfigByDestID map[descpb.ID]sqlProcessorTableConfig,
 ) (*kvRowProcessor, error) {
-	cdcEventTargets := changefeedbase.Targets{}
-	srcTablesBySrcID := make(map[descpb.ID]catalog.TableDescriptor, len(procConfigByDestID))
 	dstBySrc := make(map[descpb.ID]descpb.ID, len(procConfigByDestID))
 
 	for dstID, s := range procConfigByDestID {
 		dstBySrc[s.srcDesc.GetID()] = dstID
+	}
+
+	decoder, err := newCdcEventDecoder(ctx, procConfigByDestID, evalCtx.Settings)
+	if err != nil {
+		return nil, err
+	}
+
+	p := &kvRowProcessor{
+		cfg:      cfg,
+		spec:     spec,
+		evalCtx:  evalCtx,
+		dstBySrc: dstBySrc,
+		writers:  make(map[descpb.ID]*kvTableWriter, len(procConfigByDestID)),
+		decoder:  decoder,
+	}
+	return p, nil
+}
+
+func newCdcEventDecoder(
+	ctx context.Context,
+	procConfigByDestID map[descpb.ID]sqlProcessorTableConfig,
+	settings *cluster.Settings,
+) (cdcevent.Decoder, error) {
+	cdcEventTargets := changefeedbase.Targets{}
+	srcTablesBySrcID := make(map[descpb.ID]catalog.TableDescriptor, len(procConfigByDestID))
+
+	for _, s := range procConfigByDestID {
 		srcTablesBySrcID[s.srcDesc.GetID()] = s.srcDesc
 		cdcEventTargets.Add(changefeedbase.Target{
 			Type:              jobspb.ChangefeedTargetSpecification_EACH_FAMILY,
@@ -76,21 +102,13 @@ func newKVRowProcessor(
 
 	prefixlessCodec := keys.SystemSQLCodec
 	rfCache, err := cdcevent.NewFixedRowFetcherCache(
-		ctx, prefixlessCodec, evalCtx.Settings, cdcEventTargets, srcTablesBySrcID,
+		ctx, prefixlessCodec, settings, cdcEventTargets, srcTablesBySrcID,
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	p := &kvRowProcessor{
-		cfg:      cfg,
-		spec:     spec,
-		evalCtx:  evalCtx,
-		dstBySrc: dstBySrc,
-		writers:  make(map[descpb.ID]*kvTableWriter, len(procConfigByDestID)),
-		decoder:  cdcevent.NewEventDecoderWithCache(ctx, rfCache, false, false),
-	}
-	return p, nil
+	return cdcevent.NewEventDecoderWithCache(ctx, rfCache, false, false), nil
 }
 
 var originID1Options = &kvpb.WriteOptions{OriginID: 1}
diff --git a/pkg/crosscluster/logical/replication_statements.go b/pkg/crosscluster/logical/replication_statements.go
index aab2aa209f9..2e108b42a40 100644
--- a/pkg/crosscluster/logical/replication_statements.go
+++ b/pkg/crosscluster/logical/replication_statements.go
@@ -394,7 +394,7 @@ func newBulkSelectStatement(
 }
 
 func toParsedStatement(stmt tree.Statement) (statements.Statement[tree.Statement], error) {
-	// TODO(jeffswenson): do I have to round trip through the string or can I
-	// safely construct the statement directly?
-	return parser.ParseOne(stmt.String())
+	// User Serialize instead of String to ensure the type casts use fully
+	// qualified names.
+	return parser.ParseOne(tree.Serialize(stmt))
 }
diff --git a/pkg/crosscluster/logical/sql_crud_writer.go b/pkg/crosscluster/logical/sql_crud_writer.go
new file mode 100644
index 00000000000..a49c79ff08c
--- /dev/null
+++ b/pkg/crosscluster/logical/sql_crud_writer.go
@@ -0,0 +1,179 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package logical
+
+import (
+	"context"
+	"sort"
+
+	"github.com/cockroachdb/cockroach/pkg/ccl/changefeedccl/cdcevent"
+	"github.com/cockroachdb/cockroach/pkg/jobs/jobspb"
+	"github.com/cockroachdb/cockroach/pkg/repstream/streampb"
+	"github.com/cockroachdb/cockroach/pkg/settings/cluster"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/catalog/lease"
+	"github.com/cockroachdb/cockroach/pkg/sql/execinfra"
+	"github.com/cockroachdb/cockroach/pkg/sql/sem/eval"
+	"github.com/cockroachdb/cockroach/pkg/sql/sessiondata"
+	"github.com/cockroachdb/cockroach/pkg/sql/stats"
+	"github.com/cockroachdb/cockroach/pkg/util/tracing"
+)
+
+// sqlCrudWriter is a batch writer that implements the BatchHandler interface
+// using simple update, delete, and insert statements that assert the expected
+// previous value of the row.
+//
+// The sqlCrudWriter decodes the events, but it relies on per-table handlers to
+// process the batches.
+type sqlCrudWriter struct {
+	decoder  *eventDecoder
+	handlers map[descpb.ID]*tableHandler
+	settings *cluster.Settings
+	discard  jobspb.LogicalReplicationDetails_Discard
+}
+
+var _ BatchHandler = &sqlCrudWriter{}
+
+func newCrudSqlWriter(
+	ctx context.Context,
+	cfg *execinfra.ServerConfig,
+	evalCtx *eval.Context,
+	sd *sessiondata.SessionData,
+	discard jobspb.LogicalReplicationDetails_Discard,
+	procConfigByDestID map[descpb.ID]sqlProcessorTableConfig,
+	jobID jobspb.JobID,
+) (*sqlCrudWriter, error) {
+	decoder, err := newEventDecoder(ctx, cfg.DB, evalCtx.Settings, procConfigByDestID)
+	if err != nil {
+		return nil, err
+	}
+
+	handlers := make(map[descpb.ID]*tableHandler)
+	for dstDescID := range procConfigByDestID {
+		handler, err := newTableHandler(
+			ctx,
+			dstDescID,
+			cfg.DB,
+			cfg.Codec,
+			sd,
+			jobID,
+			cfg.LeaseManager.(*lease.Manager),
+			evalCtx.Settings,
+		)
+		if err != nil {
+			return nil, err
+		}
+		handlers[dstDescID] = handler
+	}
+
+	return &sqlCrudWriter{
+		decoder:  decoder,
+		handlers: handlers,
+		settings: evalCtx.Settings,
+		discard:  discard,
+	}, nil
+}
+
+func (c *sqlCrudWriter) HandleBatch(
+	ctx context.Context, batch []streampb.StreamEvent_KV,
+) (b batchStats, err error) {
+	ctx, sp := tracing.ChildSpan(ctx, "crudBatcher.HandleBatch")
+	defer sp.Finish()
+
+	sortedEvents, err := c.decodeAndSortEvents(ctx, batch)
+	if err != nil {
+		return batchStats{}, err
+	}
+
+	var combinedStats batchStats
+	for _, events := range eventsByTable(sortedEvents) {
+		handler := c.handlers[events[0].dstDescID]
+		stats, err := handler.handleDecodedBatch(ctx, events)
+		if err != nil {
+			return batchStats{}, err
+		}
+		stats.AddTo(&combinedStats)
+	}
+
+	return combinedStats, nil
+}
+
+// decodeAndSortEvents returns the decoded events sorted by destination
+// descriptor ID. This grouping allows `eventsByTable` to produce a single
+// group for each table.
+func (c *sqlCrudWriter) decodeAndSortEvents(
+	ctx context.Context, batch []streampb.StreamEvent_KV,
+) ([]decodedEvent, error) {
+	events := make([]decodedEvent, 0, len(batch))
+	for _, event := range batch {
+		if c.discard == jobspb.LogicalReplicationDetails_DiscardAllDeletes && len(event.KeyValue.Value.RawBytes) == 0 {
+			continue
+		}
+		decoded, err := c.decoder.decodeEvent(ctx, event)
+		if err != nil {
+			return nil, err
+		}
+		events = append(events, decoded)
+	}
+	sort.SliceStable(events, func(i, j int) bool {
+		return events[i].dstDescID < events[j].dstDescID
+	})
+	return events, nil
+}
+
+// eventsByTable is an iterator that groups events by their destination
+// descriptor ID. For optimal batching  input events should be sorted by
+// destination descriptor ID because the iterator groups runs of events with
+// the same destination.
+func eventsByTable(events []decodedEvent) func(yield func(descpb.ID, []decodedEvent) bool) {
+	return func(yield func(descpb.ID, []decodedEvent) bool) {
+		if len(events) == 0 {
+			return
+		}
+
+		start := 0
+		for i, event := range events {
+			if 1 <= i && events[i-1].dstDescID != event.dstDescID {
+				if !yield(events[start].dstDescID, events[start:i]) {
+					return
+				}
+				start = i
+			}
+		}

Leaderboard for this case

hard · 0 of 9 bots fully caught it (0% solve rate) · 0% avg coverage.

#BotResultCaughtCoverageFindingsFalse alarms
1 Claude Code (Opus 5.5) miss 0/1 0.0% 3 0
2 cursor miss 0/1 0.0% 2 0
3 Drape (Deep Review) miss 0/1 0.0% 3 0
4 Drape (Review) miss 0/1 0.0% 3 0
5 greptile miss 0/1 0.0% 3 0
6 Mjolnir (Opus 5.5) miss 0/1 0.0% 5 0
7 Mjolnir (Sonnet 5.5) miss 0/1 0.0% 2 0
8 Cubic miss 0/1 0.0% 6 1
9 qodo miss 0/1 0.0% 6 1

Bot results

Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
cdcevent DecodeKV on an empty prev value returns a deleted row that still has datums (no error), and appendDatums appends one datum per column, so prevRow is never nil or empty. attemptBatch's insert and tombstone branches therefore can't be reached on the first attempt, which forces a failed attempt and a refresh for inserts and phantom deletes. [grounded: diff] [consensus 3/3: valid=3]
uncertain
The in-place enum type override does alias the datum held by lastRow. However, the DLQ uses DebugString/ToJSON, which most likely rely on LogicalRep. The existing queryBuilder.AddRow already does the same mutation, so the claimed DLQ corruption is not shown. [uncertain: no evidence quoted] [consensus 2/5: uncertain=2, invalid=2, valid=1]
valid · impact low
AddTo counts one refresh per table batch while the legacy KV writer increments per refreshed row, so the metric undercounts relative to the other writers. The TODO acknowledges the stats are approximate. The impact is limited to metrics. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
pkg/crosscluster/logical/event_decoder.go:137
`decodeEvent` always builds a non-nil `prevRow`, even when the event has no previous value. When `PrevValue` is empty, `DecodeKV(..., cdcevent.PrevRow, ...)` returns a row marked deleted whose non-key datums are NULL, and `appendDatums` still appends one datum per column. The result is never nil or empty. `tableHandler.attemptBatch` depends on `prevRow == nil` to choose an insert and `len(prevRow) == 0` to choose a tombstone update, so neither branch ever runs on the first attempt. Every insert is sent as an `UpdateRow` with an all-NULL previous row, and every delete of a row with no previous value is sent as a `DeleteRow` with NULL columns. Both fail with a stale previous value, so any batch containing an insert or such a delete fails, is rolled back, and goes through the read refresh before it can apply. That defeats the design and roughly doubles the cost of normal ingest. If a local row really does have all-NULL non-key columns, the wrong statement can also succeed. `prevRow` should stay nil when `event.PrevValue.RawBytes` is empty or `decodedPrevRow.IsDeleted()` is true.
pkg/crosscluster/logical/event_decoder.go:165
`appendDatums` changes decoded enum datums in place (`dEnum.EnumTyp = types.Unknown`). These are the same datum pointers held by the `cdcevent.Row` that is saved in `d.lastRow` and returned by `GetLastRow()` for DLQ handling. The DLQ and any other consumer of that row then see an enum with type Unknown, which can break or garble how the DLQ entry is encoded. The datum should be copied before its type is overridden.
pkg/crosscluster/logical/table_batch_handler.go:73
`AddTo` adds 1 to `kvWriteValueRefreshes` per table batch that had any refresh, not once per refreshed row (`refreshedRows`). The legacy writers count this metric per row, so with the crud writer the refresh metrics undercount and can't be compared with the other writers.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact high
The existing KV and SQL processors call keys.StripTenantPrefix before decoding with the prefixless SystemSQLCodec decoder. The new CRUD decoder passes the raw stream key, so tenant-prefixed keys won't resolve. [grounded: diff] [consensus 5/5: valid=5]
invalid
The tombstoneUpdater builds raw KV batches without origin write options. That code is shared and unchanged, and the existing SQL writer already uses it the same way, so this gap predates the PR. [invalid: introduced=no] [consensus 5/5: invalid=5]
valid · impact medium
cdcevent.DecodeKV returns a row marked deleted, with datums, for an empty prev value. appendDatums then yields a non-nil prevRow, so attemptBatch sends inserts to UpdateRow. That fails with errStalePreviousValue, which forces the whole batch through a rollback and read refresh. [grounded: diff] [consensus 5/5: valid=5]
valid · impact low
sd comes from NewInternalSessionData, so its application name carries the internal prefix. The existing writer deliberately uses AttributedToUserInternalAppNamePrefix so its queries show up in SQL Activity; the CRUD queries won't. [grounded: diff] [consensus 5/5: valid=5]
valid · impact low
The processor calls SetSyntheticFailurePercent to inject failures, and the other writers honor it through failureInjector. The CRUD writer ignores it, and the metamorphic setting can pick it in tests. [grounded: diff] [consensus 5/5: valid=5]
valid · impact low
kvRowProcessor.Close releases its leases. The CRUD writer's Close does nothing even though its table handlers' tombstone updaters hold leases, and setupBatchHandlers/close() rely on Close for cleanup. [grounded: diff] [consensus 5/5: valid=5]
raw comments (7)
pkg/crosscluster/logical/event_decoder.go:110
<!-- cubic:v=630fea40-3b5d-4cf5-afd9-6c189af68971 --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":9} --> P1: The CRUD decoder does not strip the tenant prefix before decoding the stream key. Match the existing logical processors by calling `keys.StripTenantPrefix` and use the stripped key for both current and previous-row decoding; otherwise CRUD mode cannot decode normal tenant-prefixed events. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/event_decoder.go, line 110: <comment>The CRUD decoder does not strip the tenant prefix before decoding the stream key. Match the existing logical processors by calling `keys.StripTenantPrefix` and use the stripped key for both current and previous-row decoding; otherwise CRUD mode cannot decode normal tenant-prefixed events.</comment> <file context> @@ -37,3 +61,124 @@ type decodedEvent struct { +func (d *eventDecoder) decodeEvent( + ctx context.Context, event streampb.StreamEvent_KV, +) (decodedEvent, error) { + decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false) + if err != nil { + return decodedEvent{}, err </file context> ``` </details>
pkg/crosscluster/logical/logical_replication_writer_processor.go:762
<!-- cubic:v=1c0eb2f6-fd04-40fa-bf4b-f40a79236879 --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":7} --> P1: CRUD tombstone updates bypass the replication session override and are written without origin ID 1. Pass replication-origin session data into the tombstone updater so missing-row deletes receive the same loop-prevention metadata as inserts, updates, and deletes. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/logical_replication_writer_processor.go, line 762: <comment>CRUD tombstone updates bypass the replication session override and are written without origin ID 1. Pass replication-origin session data into the tombstone updater so missing-row deletes receive the same loop-prevention metadata as inserts, updates, and deletes.</comment> <file context> @@ -750,6 +758,11 @@ func (lrw *logicalReplicationWriterProcessor) setupBatchHandlers(ctx context.Con return err } + case writerTypeCRUD: + rp, err = newCrudSqlWriter(ctx, flowCtx.Cfg, flowCtx.EvalCtx, sd, lrw.spec.Discard, lrw.configByTable, jobspb.JobID(lrw.spec.JobID)) + if err != nil { + return err </file context> ``` </details>
pkg/crosscluster/logical/event_decoder.go:131
<!-- cubic:v=6ff9ea49-7789-4204-b09f-7fa67ff2b47a --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":8} --> P2: The CRUD decoder should preserve the absence of `PrevValue` instead of decoding it into a key-only row. Otherwise every insert without a source previous value takes the stale-update failure path before the handler can retry it as an insert. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/event_decoder.go, line 131: <comment>The CRUD decoder should preserve the absence of `PrevValue` instead of decoding it into a key-only row. Otherwise every insert without a source previous value takes the stale-update failure path before the handler can retry it as an insert.</comment> <file context> @@ -37,3 +61,124 @@ type decodedEvent struct { + prevKV.Key = event.KeyValue.Key + prevKV.Value = event.PrevValue + + decodedPrevRow, err := d.decoder.DecodeKV(ctx, prevKV, cdcevent.PrevRow, event.PrevValue.Timestamp, false) + if err != nil { + return decodedEvent{}, err </file context> ``` </details>
pkg/crosscluster/logical/table_batch_handler.go:106
<!-- cubic:v=5c031809-18a7-444c-a94f-667e72a7878a --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":9} --> P2: This labels the CRUD queries with the `$ internal` prefix, which keeps them out of SQL Activity. Use `catconstants.AttributedToUserInternalAppNamePrefix` here, matching the existing logical-replication writer. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/table_batch_handler.go, line 106: <comment>This labels the CRUD queries with the `$ internal` prefix, which keeps them out of SQL Activity. Use `catconstants.AttributedToUserInternalAppNamePrefix` here, matching the existing logical-replication writer.</comment> <file context> @@ -86,12 +100,17 @@ func newTableHandler( + // Set an applicaiton name that makes it clear which ldr job the queries + // belong to. + sessionOverride := ieOverrideBase + sessionOverride.ApplicationName = fmt.Sprintf("%s-logical-replication-%d", sd.ApplicationName, jobID) + + reader, err := newSQLRowReader(table, sessionOverride) </file context> ``` </details>
pkg/crosscluster/logical/sql_crud_writer.go:172
<!-- cubic:v=fdcf468a-d8ab-4c98-930f-b3e7f46a12b0 --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":9} --> P2: The writer processor sets this rate to inject synthetic flush failures, but this no-op makes CRUD batches ignore it. Preserve the rate and inject failures from `HandleBatch` so retry and DLQ paths can be tested. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/sql_crud_writer.go, line 172: <comment>The writer processor sets this rate to inject synthetic flush failures, but this no-op makes CRUD batches ignore it. Preserve the rate and inject failures from `HandleBatch` so retry and DLQ paths can be tested.</comment> <file context> @@ -0,0 +1,179 @@ +} + +// SetSyntheticFailurePercent implements BatchHandler. +func (c *sqlCrudWriter) SetSyntheticFailurePercent(uint32) { + +} </file context> ``` </details>
pkg/crosscluster/logical/sql_crud_writer.go:152
<!-- cubic:v=1124aa23-3ce6-475b-a079-128ec19bc186 --> <!-- cubic:review-run=72f244ed-281d-585b-b0a5-3ba4c97107f0 --> <!-- metadata:{"confidence":9} --> P2: Closing the writer leaves any tombstone descriptor leases held by its table handlers until expiry. Call `c.ReleaseLeases(ctx)` here. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At pkg/crosscluster/logical/sql_crud_writer.go, line 152: <comment>Closing the writer leaves any tombstone descriptor leases held by its table handlers until expiry. Call `c.ReleaseLeases(ctx)` here.</comment> <file context> @@ -0,0 +1,179 @@ +} + +// Close implements BatchHandler. +func (c *sqlCrudWriter) Close(ctx context.Context) { +} + </file context> ``` </details>
PR-level
<!-- cubic:review-summary:start --> **6 issues found** across 14 files <!-- cubic:review-summary:end --> <details> <summary>Prompt for AI agents (unresolved issues)</summary> ```text Check if these issues are valid — if so, understand the root cause of each and fix them. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. If appropriate, use sub-agents to investigate and fix each issue separately. <file name="pkg/crosscluster/logical/event_decoder.go"> <violation number="1" location="pkg/crosscluster/logical/event_decoder.go:110"> P1: The CRUD decoder does not strip the tenant prefix before decoding the stream key. Match the existing logical processors by calling `keys.StripTenantPrefix` and use the stripped key for both current and previous-row decoding; otherwise CRUD mode cannot decode normal tenant-prefixed events.</violation> <violation number="2" location="pkg/crosscluster/logical/event_decoder.go:131"> P2: The CRUD decoder should preserve the absence of `PrevValue` instead of decoding it into a key-only row. Otherwise every insert without a source previous value takes the stale-update failure path before the handler can retry it as an insert.</violation> </file> <file name="pkg/crosscluster/logical/table_batch_handler.go"> <violation number="1" location="pkg/crosscluster/logical/table_batch_handler.go:106"> P2: This labels the CRUD queries with the `$ internal` prefix, which keeps them out of SQL Activity. Use `catconstants.AttributedToUserInternalAppNamePrefix` here, matching the existing logical-replication writer.</violation> </file> <file name="pkg/crosscluster/logical/sql_crud_writer.go"> <violation number="1" location="pkg/crosscluster/logical/sql_crud_writer.go:152"> P2: Closing the writer leaves any tombstone descriptor leases held by its table handlers until expiry. Call `c.ReleaseLeases(ctx)` here.</violation> <violation number="2" location="pkg/crosscluster/logical/sql_crud_writer.go:172"> P2: The writer processor sets this rate to inject synthetic flush failures, but this no-op makes CRUD batches ignore it. Preserve the rate and inject failures from `HandleBatch` so retry and DLQ paths can be tested.</violation> </file> <file name="pkg/crosscluster/logical/logical_replication_writer_processor.go"> <violation number="1" location="pkg/crosscluster/logical/logical_replication_writer_processor.go:762"> P1: CRUD tombstone updates bypass the replication session override and are written without origin ID 1. Pass replication-origin session data into the tombstone updater so missing-row deletes receive the same loop-prevention metadata as inserts, updates, and deletes.</violation> </file> ``` </details> <sub>Reply with feedback, questions, or to request a fix.<br /><br />[View guided diff](https://www.cubic.dev/pr/drape-io/crbench-cubic-2026-09/pull/116?utm_source=github&utm_medium=review_footer) | [Turn on auto-fix](https://www.cubic.dev/action/auto-fix/pr/drape-io/crbench-cubic-2026-09/116?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F116) | [Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/116/ai_pr_review_github_delivery_2f3f4f897ce76ec2169f4573b2ccc57046c9a9a8eb668c347ea54372fd5c495c?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F116)</sub> <!-- cubic:review-post:ai_pr_review_github_delivery_2f3f4f897ce76ec2169f4573b2ccc57046c9a9a8eb668c347ea54372fd5c495c:52ec59e6f82819c82f3a0a4f6b165deabbd5762c:72f244ed-281d-585b-b0a5-3ba4c97107f0 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
decodeEvent always returns a non-nil prevRow, even when PrevValue is empty. attemptBatch's `case event.prevRow == nil` insert branch therefore never runs for freshly decoded events, so an insert is first tried as an UPDATE. That UPDATE fails with errStalePreviousValue and the row only lands via the refresh path. [grounded: diff] [consensus 5/5: valid=5]
valid · impact high
Both sibling writers (kvRowProcessor.processRow and sqlRowProcessor.ProcessRow) call keys.StripTenantPrefix before decoding with the prefixless SystemSQLCodec decoder. The new decodeEvent skips that step, so stream keys that carry a tenant prefix won't decode. [grounded: diff] [consensus 5/5: valid=5]
raw comments (3)
pkg/crosscluster/logical/event_decoder.go:147
### Inserts misclassified as updates **Medium Severity** <!-- DESCRIPTION START --> `decodeEvent` always builds a non-nil `prevRow`, including when the stream event has no previous value. `attemptBatch` treats a nil `prevRow` as an insert, so every insert is applied as an update first and only succeeds after a refresh. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: d1102e58-ecfa-4642-bf02-052d8a655ada --> <!-- LOCATIONS START pkg/crosscluster/logical/event_decoder.go#L126-L147 pkg/crosscluster/logical/table_batch_handler.go#L170-L187 LOCATIONS END --> <details> <summary>Additional Locations (1)</summary> - [`pkg/crosscluster/logical/table_batch_handler.go#L170-L187`](https://github.com/drape-io/crbench-cursor-2026-09/blob/da1f6cb928a3ece42e2b13e0fd2fa408fff33b2f/pkg/crosscluster/logical/table_batch_handler.go#L170-L187) </details> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90Ojk5MmM5NmNjLWUxZWEtNDljMi1hMDA3LWE4YTk3YWVmZDAxNyIsImVuY3J5cHRpb25LZXkiOiJPVTI1QnNUXzA3RW9tZzNya2VPX2ZkR3Z5NEpvczUtZ0tqQkNabmtfbmkwIiwiYnJhbmNoIjoicHItZDA2NjBkOTQ0YSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90Ojk5MmM5NmNjLWUxZWEtNDljMi1hMDA3LWE4YTk3YWVmZDAxNyIsImVuY3J5cHRpb25LZXkiOiJPVTI1QnNUXzA3RW9tZzNya2VPX2ZkR3Z5NEpvczUtZ0tqQkNabmtfbmkwIiwiYnJhbmNoIjoicHItZDA2NjBkOTQ0YSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjExNiwiY29tbWl0U2hhIjoiZGExZjZjYjkyOGEzZWNlNDJlMmIxM2UwZmQyZmE0MDhmZmYzM2IyZiIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit da1f6cb928a3ece42e2b13e0fd2fa408fff33b2f. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
pkg/crosscluster/logical/event_decoder.go:131
### Tenant prefix not stripped before decode **High Severity** <!-- DESCRIPTION START --> `decodeEvent` passes stream keys to `DecodeKV` without stripping the tenant prefix. The shared CDC decoder uses `SystemSQLCodec`, so keys from a secondary tenant fail to decode and the CRUD writer cannot apply rows. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 0b24b308-a5ef-4201-b476-8d0a3e3f7eaf --> <!-- LOCATIONS START pkg/crosscluster/logical/event_decoder.go#L109-L131 LOCATIONS END --> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjFmZGI4NjM4LTU0OWUtNDIzOC1iNDM3LTEzMjA1MzlhMTllNCIsImVuY3J5cHRpb25LZXkiOiJoaTdlRTNZV1RXS09LRVJlNUtWY0xITlEtaGxrUGdnR3lVRTE4SnZjVV9jIiwiYnJhbmNoIjoicHItZDA2NjBkOTQ0YSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjFmZGI4NjM4LTU0OWUtNDIzOC1iNDM3LTEzMjA1MzlhMTllNCIsImVuY3J5cHRpb25LZXkiOiJoaTdlRTNZV1RXS09LRVJlNUtWY0xITlEtaGxrUGdnR3lVRTE4SnZjVV9jIiwiYnJhbmNoIjoicHItZDA2NjBkOTQ0YSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjExNiwiY29tbWl0U2hhIjoiZGExZjZjYjkyOGEzZWNlNDJlMmIxM2UwZmQyZmE0MDhmZmYzM2IyZiIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit da1f6cb928a3ece42e2b13e0fd2fa408fff33b2f. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
PR-level
<!-- BUGBOT_REVIEW --> Cursor Bugbot has reviewed your changes using high effort and found 2 potential issues. <!-- BUGBOT_FIX_ALL --> <a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aTo2YmNlZmRhMS0wNGQyLTQ3YTMtOGYyMy1kYmI5ODZiYjBmZjUiLCJlbmNyeXB0aW9uS2V5IjoiUnlYWE85TnRpLXVYOEhRZTJoMGtQMWEyRHhOVUJNemRhVGsxeFMwUEVuYyIsImJyYW5jaCI6InByLWQwNjYwZDk0NGEiLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <!-- /BUGBOT_FIX_ALL --> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN --> <sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END --> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit da1f6cb928a3ece42e2b13e0fd2fa408fff33b2f. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact high
The new crud writer passes the raw key to a decoder built with keys.SystemSQLCodec, but never calls keys.StripTenantPrefix. Both sibling writers do call it (lww_kv_processor processRow and sqlRowProcessor.ProcessRow), so streams from a tenant would fail to decode. [grounded: tree pkg/crosscluster/logical/event_decoder.go] [consensus 3/3: valid=3]
valid · impact medium
prevRow is always filled with key columns plus NULLs, even when PrevValue is absent, so it is never nil. attemptBatch then sends inserts to UpdateRow and tombstone deletes to DeleteRow. Both fail as stale, which forces the read-refresh slow path every time. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
On a decode error, decodeEvent returns without resetting lastRow, so GetLastRow returns the previous event's row. The sibling processors reset lastRow on decode errors. The real-world impact is limited because decode errors often have no pg code and so cannot be DLQ'd at all. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
pkg/crosscluster/logical/event_decoder.go:108
**Tenant-prefixed keys fail to decode in crud writer** `decodeEvent` hands the raw stream key to a decoder built with the system (prefixless) codec, but never strips the tenant prefix like the sql and kv writers do. Any stream from a non-system tenant fails to decode. Call keys.StripTenantPrefix on event.KeyValue.Key before decoding (and use the stripped key for prevKV). ``` pkg/crosscluster/logical/event_decoder.go:108 ctx context.Context, event streampb.StreamEvent_KV, pkg/crosscluster/logical/event_decoder.go:109 ) (decodedEvent, error) { pkg/crosscluster/logical/event_decoder.go:110 decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false) pkg/crosscluster/logical/event_decoder.go:111 if err != nil { pkg/crosscluster/logical/lww_kv_processor.go:103 prefixlessCodec := keys.SystemSQLCodec pkg/crosscluster/logical/lww_kv_processor.go:104 rfCache, err := cdcevent.NewFixedRowFetcherCache( pkg/crosscluster/logical/lww_kv_processor.go:166 keyValue.Key, err = keys.StripTenantPrefix(keyValue.Key) pkg/crosscluster/logical/lww_kv_processor.go:167 if err != nil { pkg/crosscluster/logical/lww_kv_processor.go:168 return batchStats{}, errors.Wrap(err, "stripping tenant prefix") pkg/crosscluster/logical/lww_kv_processor.go:169 } ```
pkg/crosscluster/logical/event_decoder.go:135
**Inserts and tombstones never take their own path** `prevRow` is decoded even when the event has no previous value, giving keys plus NULLs instead of nil. So `attemptBatch` sends every insert to UpdateRow, which fails and forces a rollback and read refresh. Leave `prevRow` nil unless `event.PrevValue.IsPresent()`. ``` pkg/crosscluster/logical/event_decoder.go:127 var prevKV roachpb.KeyValue pkg/crosscluster/logical/event_decoder.go:128 prevKV.Key = event.KeyValue.Key pkg/crosscluster/logical/event_decoder.go:129 prevKV.Value = event.PrevValue pkg/crosscluster/logical/event_decoder.go:131 decodedPrevRow, err := d.decoder.DecodeKV(ctx, prevKV, cdcevent.PrevRow, event.PrevValue.Timestamp, false) pkg/crosscluster/logical/event_decoder.go:57 // prevRow is either the previous row from the replication stream or it is pkg/crosscluster/logical/event_decoder.go:58 // the local version of the row if there was a read refresh. pkg/crosscluster/logical/event_decoder.go:59 // pkg/crosscluster/logical/event_decoder.go:60 // The datums in prevRow are in col id order for the destination table. nil pkg/crosscluster/logical/event_decoder.go:61 // prevRow may still lose LWW if there is a recent tombstone. pkg/crosscluster/logical/event_decoder.go:62 prevRow tree.Datums ```
pkg/crosscluster/logical/event_decoder.go:110
**DLQ logs wrong row after decode error** When decoding fails, `decodeEvent` does not clear `lastRow`, so the DLQ entry for the failed event gets the previous event's row. The other writers reset it on a decode error. Set `d.lastRow = cdcevent.Row{}` at the start of `decodeEvent`. ``` pkg/crosscluster/logical/event_decoder.go:110 decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false) pkg/crosscluster/logical/event_decoder.go:111 if err != nil { pkg/crosscluster/logical/event_decoder.go:112 return decodedEvent{}, err pkg/crosscluster/logical/event_decoder.go:113 } pkg/crosscluster/logical/lww_row_processor.go:355 row, err := srp.decoder.DecodeKV(ctx, kv, cdcevent.CurrentRow, kv.Value.Timestamp, false) pkg/crosscluster/logical/lww_row_processor.go:356 if err != nil { pkg/crosscluster/logical/lww_row_processor.go:357 srp.lastRow = cdcevent.Row{} pkg/crosscluster/logical/lww_row_processor.go:358 return batchStats{}, errors.Wrap(err, "decoding KeyValue") pkg/crosscluster/logical/lww_kv_processor.go:171 row, err := p.decoder.DecodeKV(ctx, keyValue, cdcevent.CurrentRow, keyValue.Value.Timestamp, false) pkg/crosscluster/logical/lww_kv_processor.go:172 if err != nil { ```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
cdcevent decodes an absent value as a deleted row whose key columns are filled in and other columns are NULL, so prevRow is never empty. attemptBatch then sends inserts through UpdateRow and pk-only deletes through DeleteRow. Both fail and every such batch goes through the refresh path. The refresh path recovers, but the insert and tombstone fast paths never run. [grounded: tree pkg/crosscluster/logical/event_decoder.go] [consensus 3/3: valid=3]
valid · impact medium
The decoder is built with the prefixless keys.SystemSQLCodec, and both existing writers call keys.StripTenantPrefix before decoding. The new CRUD decodeEvent does not, so keys streamed from a secondary-tenant source fail to decode. [grounded: tree pkg/crosscluster/logical/event_decoder.go] [consensus 3/3: valid=3]
valid · impact low
lastRow is only assigned after a successful decode and is never cleared on error. If decoding fails, GetLastRow returns the previous event's row, which maybeDLQ may log or persist. The sibling processors reset lastRow on decode errors. Impact is limited because many decode errors are not eligible for the DLQ. [grounded: tree pkg/crosscluster/logical/event_decoder.go] [consensus 3/3: valid=3]
raw comments (3)
pkg/crosscluster/logical/event_decoder.go:131
**Missing previous value still yields non-nil prevRow** `decodeEvent` always decodes `event.PrevValue` and builds `prevRow` with `make`+append, even with no previous value, so `prevRow` is never nil or empty. `attemptBatch` relies on that to choose insert or tombstone update. Only decode and fill `prevRow` when `event.PrevValue.IsPresent()`, as `kvRowProcessor.addToBatch` does. Otherwise leave it nil. ``` pkg/crosscluster/logical/event_decoder.go:127 var prevKV roachpb.KeyValue pkg/crosscluster/logical/event_decoder.go:128 prevKV.Key = event.KeyValue.Key pkg/crosscluster/logical/event_decoder.go:129 prevKV.Value = event.PrevValue pkg/crosscluster/logical/event_decoder.go:131 decodedPrevRow, err := d.decoder.DecodeKV(ctx, prevKV, cdcevent.PrevRow, event.PrevValue.Timestamp, false) pkg/crosscluster/logical/lww_kv_processor.go:340 if prevValue.IsPresent() { pkg/crosscluster/logical/lww_kv_processor.go:341 if err := w.updateRow(ctx, b, prevRow, row); err != nil { pkg/crosscluster/logical/lww_kv_processor.go:342 return err pkg/crosscluster/logical/lww_kv_processor.go:343 } pkg/crosscluster/logical/lww_kv_processor.go:344 } else { pkg/crosscluster/logical/lww_kv_processor.go:345 if err := w.insertRow(ctx, b, row); err != nil { ```
pkg/crosscluster/logical/event_decoder.go:102
**CRUD writer doesn't strip tenant prefix before decoding** `decodeEvent` passes `event.KeyValue` to the decoder as-is. The decoder uses the prefixless system codec, and the other two writers call `keys.StripTenantPrefix` first. Stream keys carry a tenant prefix, so decoding fails. Strip the tenant prefix from the key before calling DecodeKV, for both the current-row and prev-row decodes. ``` pkg/crosscluster/logical/event_decoder.go:101 return &eventDecoder{ pkg/crosscluster/logical/event_decoder.go:102 decoder: decoder, pkg/crosscluster/logical/event_decoder.go:103 srcToDest: srcToDest, pkg/crosscluster/logical/event_decoder.go:104 }, nil pkg/crosscluster/logical/lww_kv_processor.go:166 keyValue.Key, err = keys.StripTenantPrefix(keyValue.Key) pkg/crosscluster/logical/lww_row_processor.go:350 kv.Key, err = keys.StripTenantPrefix(kv.Key) ```
pkg/crosscluster/logical/event_decoder.go:97
**lastRow is not reset when decoding fails** When `decodeEvent` fails to decode, `d.lastRow` keeps the previous event's row. The other processors reset it on decode error. `maybeDLQ` then uses `GetLastRow()` and writes the wrong row to the DLQ. Set d.lastRow = cdcevent.Row{} on each decode error return. ``` pkg/crosscluster/logical/event_decoder.go:96 decoder, err := newCdcEventDecoder(ctx, procConfigByDestID, settings) pkg/crosscluster/logical/event_decoder.go:97 if err != nil { pkg/crosscluster/logical/event_decoder.go:98 return nil, err pkg/crosscluster/logical/event_decoder.go:99 } pkg/crosscluster/logical/lww_kv_processor.go:173 p.lastRow = cdcevent.Row{} pkg/crosscluster/logical/lww_row_processor.go:357 srp.lastRow = cdcevent.Row{} ```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
prevRow is always a non-nil slice, even when PrevValue is empty. attemptBatch then sends inserts down the `event.prevRow != nil` UpdateRow branch. That update fails, which forces a refresh read and a retry, so ordinary inserts take a real performance hit. [grounded: tree pkg/crosscluster/logical/event_decoder.go] [consensus 3/3: valid=3]
valid · impact low
Close is empty, while its sibling kvRowProcessor.Close calls ReleaseLeases. setupBatchHandlers and processor close call only Close, so leases acquired by the tombstone updater can leak until they expire. [grounded: tree pkg/crosscluster/logical/sql_crud_writer.go] [consensus 3/3: valid=3]
valid · impact low
The method ignores the rate, and HandleBatch never injects failures. The FailureRate knob and debug failure injection therefore do nothing for the crud writer, which is now a metamorphic test choice. [grounded: tree pkg/crosscluster/logical/sql_crud_writer.go] [consensus 3/3: valid=3]
raw comments (4)
pkg/crosscluster/logical/event_decoder.go:137
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Inserts take the slow path** `decodeEvent` always fills `prevRow`, even when `PrevValue` is empty or represents a tombstone. For inserts into an empty destination, `tableHandler` therefore attempts an update, fails, reads the destination, and only then inserts. This adds failed writes and reads to ordinary insert batches. Leave `prevRow` nil when `decodedPrevRow.IsDeleted()` is true so these events take the insert path.
pkg/crosscluster/logical/sql_crud_writer.go:153
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Stopped writers keep table leases** `Close` does not release the leases owned by the table handlers. A tombstone write acquires and retains a lease, but processor shutdown and worker-pool replacement call only `Close`; they need not reach another checkpoint. The discarded writer can then hold leases until they expire, delaying table changes. Call `c.ReleaseLeases(ctx)` here. ```suggestion func (c *sqlCrudWriter) Close(ctx context.Context) { c.ReleaseLeases(ctx) } ```
pkg/crosscluster/logical/sql_crud_writer.go:174
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **Requested failures never happen** `SetSyntheticFailurePercent` ignores its argument, so the processor's `FailureRate` testing knob and requested debug failures do nothing when `crud` is selected. The setting now includes `crud` in its test choices, so retry tests can run without the failures they requested. Store the rate and check it in `HandleBatch`, using the existing `failureInjector`.
PR-level
<!-- greptile_summary --> <h2><a href="https://app.greptile.com/api/retrigger?id=76825648"><picture><source media="(prefers-color-scheme: dark)" srcset="https://greptile-static-assets.s3.amazonaws.com/badges/RetriggerDark.svg?v=2"><source media="(prefers-color-scheme: light)" srcset="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2"><img alt="Retrigger" src="https://greptile-static-assets.s3.amazonaws.com/badges/Retrigger.svg?v=2" align="right"></picture></a>Confidence Score: 4/5</h2> **[High risk]** Adds new replication writer implementation with cross\-cluster data flow changes\. The PR appears safe to merge, with non-blocking fixes for unnecessary insert work, retained leases, and ignored test failures. <h2>Findings</h2> 1. <img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top">&nbsp;**Inserts take the slow path** <a href="https://github.com/drape-io/crbench-greptile-2026-09/pull/116#discussion_r4214152346">▶</a> 2. <img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top">&nbsp;**Stopped writers keep table leases** <a href="https://github.com/drape-io/crbench-greptile-2026-09/pull/116#discussion_r4214152354">▶</a> 3. <img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top">&nbsp;**Requested failures never happen** <a href="https://github.com/drape-io/crbench-greptile-2026-09/pull/116#discussion_r4214152365">▶</a> <h3>Summary</h3> Adds an optional `crud` writer for immediate logical replication. It decodes stream events, groups them by destination table, and uses explicit inserts, updates, and deletes. - Shares the event-decoder setup with the KV writer. - Passes job-specific session settings to SQL reads and writes. - Uses serializable SQL formatting for type references. - Three non-blocking fixes would avoid unnecessary insert work, release leases on close, and preserve requested test failures. <details open><summary>Diagram</summary> ```mermaid %%{init: {'theme': 'neutral'}}%% flowchart TD A[Stream events] --> B[Decode current and previous rows] B --> C[Group by destination table] C --> D[Attempt table writes] D -->|Failure| E[Read destination rows] E --> F[Skip older events and retry writes] F -->|Failure| G[Processor retries events individually] ``` </details> <!-- greptile_confidence_score:4 --> <sub>Reviews (1) · Last reviewed commit: ["Update pkg/crosscluster/logical/BUILD.ba..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/1292ba305792d874a17220ae5b13a2b6b733bf36) · [Reviewed by Greptile](https://www.greptile.com/?utm_source=greptile_expert&utm_medium=github&utm_campaign=code_reviews&utm_content=drape-io/crbench-greptile-2026-09)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
In cdcevent, decodeKV returns a Row with non-nil datums even for an empty value, because nextRow asserts that datums are non-nil and marks the row deleted. So prevRow is always full-length. attemptBatch's prevRow == nil insert branch and its len(prevRow)==0 tombstone branch are unreachable on the first attempt. Inserts and deletes without a prior value fail with a stale-value error and force a refresh and retry. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
lastRow is only assigned after DecodeKV, the table lookup and appendDatums all succeed, so a failure leaves the previous event's row in place. GetLastRow then passes that stale row to maybeDLQ. sqlRowProcessor resets lastRow when decoding fails. [grounded: diff] [consensus 3/4: valid=3, invalid=1]
valid · impact high
The decoder uses the prefixless keys.SystemSQLCodec. Both existing writers call keys.StripTenantPrefix before DecodeKV, but the new CRUD path passes the raw key. Keys from a secondary-tenant source therefore fail to decode or resolve to the wrong table. [grounded: diff] [consensus 3/3: valid=3]
valid · impact medium
Same as finding 0: the prev row is decoded even when PrevValue is empty and yields a full-length key-plus-NULLs row. attemptBatch therefore always takes the update or delete path, fails with errStalePreviousValue, and falls back to refresh and retry. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
Duplicate of finding 1: when decoding fails, lastRow keeps the previous event's row, so maybeDLQ records an unrelated row. [grounded: diff] [consensus 3/4: valid=3, invalid=1]
raw comments (5)
pkg/crosscluster/logical/event_decoder.go:127
[P2] Missing previous value still produces a full-length `prevRow`, so inserts are sent down the update path `decodeEvent` always decodes `event.PrevValue` and builds `prevRow`, even when there is no previous value (empty `RawBytes`). In that case `cdcevent` returns a "deleted" row whose datums are non-nil: the key columns plus NULLs. So `prevRow` is never nil or empty. But `tableHandler.attemptBatch` uses `prevRow == nil` to pick `InsertRow` and `len(prevRow) == 0` to pick the tombstone update. With this decoder, every replicated insert goes to `UpdateRow` with an all-NULL prior row, and every delete with no prior value goes to `DeleteRow`. These usually affect 0 rows, return `errStalePreviousValue`, abort the whole batch transaction, and force the read-refresh and a second attempt. Any batch containing an insert (the common case under insert-heavy load) therefore always pays for a failed transaction plus a refresh read. The other writers handle this by checking `prevValue.IsPresent()` or `RawBytes == nil`. `prevRow` should stay nil when there is no previous value.
pkg/crosscluster/logical/event_decoder.go:109
[P3] `lastRow` keeps the previous event's row when decoding fails, so the DLQ can record the wrong row `d.lastRow` is set only after `DecodeKV`, the `srcToDest` lookup and `appendDatums` all succeed. If any of them fails, `lastRow` still holds the row from the previously decoded event. When the processor retries events one at a time and then DLQs the failing one, it calls `bh.GetLastRow()`, so the DLQ entry gets the table, key and contents of a different event. The existing `sqlRowProcessor` resets `lastRow = cdcevent.Row{}` when decoding fails for this reason. This only happens on decode failures, which are rare, but when it does the DLQ entry is misleading.
pkg/crosscluster/logical/event_decoder.go:108
[P1] CRUD writer decodes keys without stripping the tenant prefix `decodeEvent` passes `event.KeyValue` straight to `d.decoder.DecodeKV`. The decoder comes from `newCdcEventDecoder`, which uses the prefixless `keys.SystemSQLCodec`. Both existing writers call `keys.StripTenantPrefix` on the key before decoding (`lww_kv_processor.go` `processRow` and `lww_row_processor.go` `ProcessRow`), but this path never does. The prev-value decode reuses the same unstripped key. When the source cluster is a secondary/application tenant, its keys carry a tenant prefix, so decoding either fails or resolves the wrong table ID. With `immediate_mode_writer = 'crud'`, every event fails and ends up in the retry queue or the DLQ. The setting is also picked metamorphically in tests, so tests that run against a test tenant would hit this too.
pkg/crosscluster/logical/event_decoder.go:125
[P2] Missing previous value is decoded into a non-empty prevRow, so every insert takes the update path When `event.PrevValue` is empty (a fresh insert, or a delete with no prior value), `DecodeKV(..., cdcevent.PrevRow, ...)` still returns an initialized deleted row. `appendDatums` then fills `prevRow` with one datum per column: the decoded key plus NULLs. `prevRow` is therefore never nil or empty. As a result, `tableHandler.attemptBatch` never reaches its `prevRow == nil` insert branch or its `isDelete && len(prevRow) == 0` tombstone branch. Plain inserts run `UpdateRow` with a `(pk, NULL, …)` previous value, and deletes with no prior value run `DeleteRow`. Both usually affect 0 rows and return `errStalePreviousValue`. That aborts the whole batch transaction and forces a read refresh plus a second attempt for any batch containing a new row. The fallback in `refreshPrevRows` keeps the result correct, but the fast path is effectively lost for insert-heavy workloads. `prevRow` should be left nil when `!event.PrevValue.IsPresent()`, as the KV writer does when it checks `prevValue.IsPresent()`.
pkg/crosscluster/logical/event_decoder.go:110
[P3] `lastRow` is not reset when decoding fails, so the DLQ records a stale row `d.lastRow` is only assigned after the current row decodes successfully. If `DecodeKV`, the table lookup, or `appendDatums` fails, `lastRow` keeps the previous event's row. When a single-event batch fails this way, the processor calls `maybeDLQ(ctx, batch[i], bh.GetLastRow(), ...)`, which logs and DLQs the event with an unrelated row from a different key or table. The existing processors set `lastRow = cdcevent.Row{}` on decode error to avoid exactly this.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact medium
decodeEvent always decodes PrevValue. cdcevent's decodeKV returns a deleted row that still has a datum for every column, so prevRow is never nil or empty. As a result, attemptBatch sends inserts to UpdateRow and deletes with no previous value to DeleteRow. Both fail with errStalePreviousValue and force the refresh path. [grounded: diff] [consensus 3/3: valid=3]
valid · impact medium
This is the same mechanism as finding 0. Because prevRow is always populated with n datums, the insert and tombstone-update branches of attemptBatch can never run on the first attempt. Every such batch therefore costs a failed transaction plus a read refresh. [grounded: diff] [consensus 3/3: valid=3]
raw comments (2)
pkg/crosscluster/logical/event_decoder.go:127
[P1] prevRow is always populated, so inserts and tombstone deletes are misclassified `decodeEvent` always decodes `event.PrevValue` and returns a non-nil `prevRow` with `len(dstTable.columns)` entries. This holds even when there is no previous value (`!event.PrevValue.IsPresent()`). For an absent value the row fetcher yields a tombstone whose key columns are filled in and whose other columns are NULL. `tableHandler.attemptBatch` relies on `prevRow == nil` or empty to select the insert path (`event.prevRow == nil`) and the tombstone-update path (`isDelete && len(prevRow)==0`). Neither condition can ever be true for events from this decoder. - A replicated insert goes down the `UpdateRow` branch with a fake `(key, NULL…)` prev row. - A delete of a row with no prior value goes down `DeleteRow` instead of `updateTombstone`. Both fail with `errStalePreviousValue`, so every first attempt of such a batch fails and falls back to `refreshPrevRows` plus a second transaction. The legacy KV path avoids this by checking `prevValue.IsPresent()` before using the prev row. Leave `prevRow` nil when `event.PrevValue` is not present.
pkg/crosscluster/logical/event_decoder.go:110
[P2] Decoded events never have a nil prevRow, so inserts are routed to UpdateRow decodeEvent always builds prevRow with `make(tree.Datums, 0, n)` and fills it from `appendDatums`. The previous value is decoded with `DecodeKV(..., cdcevent.PrevRow, ...)`. When the source event has no previous value (a plain insert, or a delete with no prior row), that decode returns a tombstone-style row. The row has the right number of datums, with the key columns filled in and the other columns NULL. So `prevRow` is non-nil and has non-zero length. `tableHandler.attemptBatch` relies on `event.prevRow == nil` for inserts and on `len(event.prevRow) == 0` for tombstone updates. For events from this decoder, the insert case is never taken. Inserts go to `UpdateRow` with an all-NULL prev row, and deletes without a prior value go to `DeleteRow` with that same prev row. These calls fail the previous-value check, or in rare cases match a row whose non-key columns are all NULL. The batch then falls back to `refreshPrevRows` and a second attempt, which only gets the right `prevRow` from the read. This happens whenever the 'crud' writer processes inserts, so every insert batch costs an extra failed transaction and a read. It also makes the insert and tombstone-update paths unreachable in practice. Setting `prevRow` to nil when `event.PrevValue.RawBytes` is empty (or the decoded prev row is deleted) would fix it.
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
valid · impact high
Both the KV and SQL processors call keys.StripTenantPrefix before decoding with the prefixless SystemSQLCodec decoder. The new eventDecoder skips that step, so keys from tenant streams won't decode. [grounded: diff] [consensus 3/3: valid=3]
invalid
The tombstoneUpdater and its use in tableHandler were already there before this diff and were not changed. The existing lwwQuerier uses the same tombstone path in the same way, so any origin-ID gap was already present. [invalid: introduced=no] [consensus 3/4: invalid=3, uncertain=1]
valid · impact medium
decodeEvent always fills prevRow, even when PrevValue is empty. As a result, attemptBatch's insert branch (prevRow == nil) and its tombstone branch are never taken on the first attempt. Inserts become updates/deletes against all-NULL rows, which fail and force a read-refresh retry. [grounded: diff] [consensus 3/3: valid=3]
valid · impact medium
lastRow is set only after a successful decode and is never cleared on failure, unlike the kv and sql processors, which reset it. On a single-event retry that fails to decode, GetLastRow returns the previous event's row, and that stale row is what gets passed to the DLQ. [grounded: diff] [consensus 3/4: valid=3, uncertain=1]
valid · impact low
Close is empty, so the tombstone updater's leases stay held after shutdown or a pool resize until they expire. kvRowProcessor's Close, by contrast, calls ReleaseLeases. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
The method is empty and HandleBatch never injects a failure, so the FailureRate testing knob and the debug flush failure injection do nothing when the crud writer is selected. [grounded: diff] [consensus 3/3: valid=3]
raw comments (8)
pkg/crosscluster/logical/event_decoder.go:110
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 1\. Tenant streams fail with the crud writer <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>eventDecoder.decodeEvent</i></b> passes the source key directly to a CDC decoder configured with the system SQL codec, without removing its tenant prefix. For non-system tenant streams, decoding fails before either the current or previous row can be applied. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description The CRUD decoder passes tenant-prefixed stream keys to a decoder using the system SQL codec, which rejects them. ## Fix Focus Areas - pkg/crosscluster/logical/event_decoder.go[107-131] ## Recommended Fix Strip the tenant prefix once before decoding, and use the stripped key for both the current and previous values. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/8bfe79ab-12d5-4c05-987d-824e4e819ffb?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/8bfe79ab-12d5-4c05-987d-824e4e819ffb)
pkg/crosscluster/logical/sql_row_writer.go:42
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 2\. Tombstone deletes can loop back <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>sqlRowWriter.getExecutorOverride</i></b> sets origin ID 1 for SQL writes, but the CRUD handler&#x27;s tombstone path uses a separate <b><i>tombstoneUpdater</i></b> constructed with session data whose origin ID remains zero. When that path writes a replicated tombstone, the producer&#x27;s origin-zero rangefeed filter can admit the write as a local event. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description The new CRUD SQL writes carry the replication origin ID, but the handler's KV tombstone writes do not. ## Fix Focus Areas - pkg/crosscluster/logical/table_batch_handler.go[103-118] - pkg/crosscluster/logical/sql_row_writer.go[37-43] ## Recommended Fix Ensure the session data used to construct the CRUD tombstone deleter carries origin ID 1, while retaining the per-event origin timestamp. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/526f7a63-7c5d-4919-9b60-9fda690fbc86?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/526f7a63-7c5d-4919-9b60-9fda690fbc86)
pkg/crosscluster/logical/event_decoder.go:140
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 3\. Every insert takes a failed transaction first <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>decodeEvent</i></b> runs <b><i>appendDatums</i></b> on <b><i>PrevRow</i></b> even when <b><i>event.PrevValue.RawBytes</i></b> is empty, producing a non-nil <b><i>prevRow</i></b> from a deleted row with key columns and NULLs elsewhere. Inserts and deletes without a previous value therefore miss <b><i>attemptBatch</i></b>’s insert and tombstone branches, attempt an UPDATE or DELETE against that row, and require a read-refresh and second transaction after the batch fails. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description `eventDecoder.decodeEvent` fills `prevRow` when the source event has no previous value, causing the table handler to miss its insert and tombstone branches on the first attempt. ## Fix Focus Areas - pkg/crosscluster/logical/event_decoder.go[127-148] ## Recommended Fix If `len(event.PrevValue.RawBytes) == 0`, skip decoding the previous row and leave `prevRow` nil. Also leave it nil if the decoded previous row reports `IsDeleted()`; otherwise decode it as today. Add a unit test verifying that an event with no previous value produces `prevRow == nil`. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/a0369cfc-0f03-485e-8959-d4e861a309b8?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/a0369cfc-0f03-485e-8959-d4e861a309b8)
pkg/crosscluster/logical/event_decoder.go:125
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> 4\. Failed rows can land in another table's dlq <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>decodeEvent</i></b> updates <b><i>d.lastRow</i></b> only after a successful decode and does not clear it when <b><i>DecodeKV</i></b> or <b><i>appendDatums</i></b> fails. When a single-event decode fails, <b><i>GetLastRow</i></b> can return the previous event’s row, which <b><i>maybeDLQ</i></b> and <b><i>dlqClient.Log</i></b> use for the DLQ table, row JSON and mutation type alongside the failing event’s KV bytes. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description A failed decode can leave `eventDecoder.lastRow` pointing to an earlier event, causing a single-event DLQ record to use the wrong row or table. ## Fix Focus Areas - pkg/crosscluster/logical/event_decoder.go[107-125] ## Recommended Fix Clear `d.lastRow` to `cdcevent.Row{}` at the start of each `decodeEvent` attempt. Assign `d.lastRow = decodedRow` immediately after `DecodeKV` succeeds, so an `appendDatums` failure still leaves the current event’s row available. This matches the clearing behavior of `sqlRowProcessor.ProcessRow`. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/57161b77-2bcd-42a6-acd1-983cc9b467c6?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/57161b77-2bcd-42a6-acd1-983cc9b467c6)
pkg/crosscluster/logical/sql_crud_writer.go:153
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> 5\. Writer shutdown leaves descriptor leases held <code>🐞 Bug</code> <code>☼ Reliability</code> <pre> <b><i>sqlCrudWriter.Close</i></b> does nothing, leaving descriptor leases held by table handlers’ tombstone updaters unreleased. When the processor shuts down or <b><i>setupBatchHandlers</i></b> closes old handlers to resize the pool before another successful checkpoint, those leases remain held until they expire and can hold up schema changes on replicated tables. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description `sqlCrudWriter.Close` leaves tombstone-updater descriptor leases held when the processor shuts down or old handlers are closed during pool resizing, including when an error prevents another successful checkpoint. ## Fix Focus Areas - pkg/crosscluster/logical/sql_crud_writer.go[151-164] ## Recommended Fix Have `sqlCrudWriter.Close` call `c.ReleaseLeases(ctx)`, as `kvRowProcessor.Close` does, so it releases every table handler’s leases. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/c2ce1179-9718-4c5e-8839-82cbd15bd9f4?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/c2ce1179-9718-4c5e-8839-82cbd15bd9f4)
pkg/crosscluster/logical/sql_crud_writer.go:174
<img src="https://img.shields.io/badge/Low-634FD1?style=flat-square" height="20px" alt="Informational"> 6\. Debug failure injection does nothing with the new writer <code>🐞 Bug</code> <code>◔ Observability</code> <pre> <b><i>sqlCrudWriter.SetSyntheticFailurePercent</i></b> is an empty method and does not embed <b><i>failureInjector</i></b>, so <b><i>HandleBatch</i></b> never returns <b><i>errInjected</i></b>. The streaming testing knob <b><i>FailureRate</i></b> and the operator debug flush hook (<b><i>RecordFlushStart</i></b>) both stop working whenever the crud writer is selected, including when the metamorphic setting picks it in tests. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description The crud writer ignores synthetic failure injection. ## Fix Focus Areas - pkg/crosscluster/logical/sql_crud_writer.go[31-36] - pkg/crosscluster/logical/sql_crud_writer.go[80-89] - pkg/crosscluster/logical/sql_crud_writer.go[171-174] ## Recommended Fix Embed `failureInjector` in `sqlCrudWriter` and remove the empty `SetSyntheticFailurePercent`. At the start of `HandleBatch`, call `injectFailure()` and return its error if there is one, as the SQL processor does. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details> [Dismiss ↗](https://app.qodo.ai/findings/bcf7bf2d-e58e-46ce-bce3-afa09a80b620?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/bcf7bf2d-e58e-46ce-bce3-afa09a80b620)
PR-level
<h3>PR Summary by Qodo</h3> Add an optional SQL CRUD writer for logical replication <code>✨ Enhancement</code> <code>🐞 Bug fix</code> <code>🧪 Tests</code> <code>🕐 40+ Minutes</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <details> <summary>AI Description</summary> <dl> <dd> <br/> ><pre> >• Add an optional CRUD writer that applies logical replication events using conditional SQL inserts, > updates, and deletes. >• Decode source events into destination-column order and refresh stale previous rows before retrying > writes. >• Pass job-specific session settings to internal SQL operations and serialize fully qualified > statement casts. ></pre> </dd> </dl> </details> <details> <summary>Diagram</summary> <dl> <dd> <br/> ```mermaid graph TD Stream["KV stream"] --> Processor["Writer processor"] --> CRUD["CRUD batch writer"] --> Decoder["Event decoder"] --> Handler["Table handler"] --> Operations["SQL row operations"] --> Destination[("Destination table")] ``` </dd> </dl> </details> <details> <summary>High-Level Assessment</summary> <dl> <dd> <br/> >The following are alternative approaches to this PR: <details> <summary><b>1. Extend the existing SQL upsert writer</b></summary> <dl> <dd> - ➕ Reuses the established writer path and its operational behavior. - ➖ Does not naturally express the new conditional insert, update, delete, and previous-row refresh strategy. </dd> </dl> </details> >**Recommendation:** Keep CRUD as a separately selectable writer while its conflict-handling behavior is evaluated; this limits changes to the existing SQL and legacy KV paths. Add focused end-to-end coverage for CRUD before making it the default, since the changed tests do not explicitly select it. </dd> </dl> </details> <details> <summary> Files changed (14) <code> +426 / -41 </code> </summary> <dl> <dd> <br/> <details> <summary>Enhancement (6) <code> +383 / -19 </code></summary> <dl> <dd> <details> <summary>event_decoder.go<code>Decode source KVs into destination-table datums</code> <code>+145/-0</code></summary> <br/> >Decode source KVs into destination-table datums > ><pre> >• Adds destination descriptor mapping and decoding of current and previous KV values into destination column order. Handles enum datum types for SQL execution and retains the last decoded row for batch error reporting. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-0c9b889c25d696f75fc06cedd49c767700b219f3034eedaf7e6706de6395a5ec'>pkg/crosscluster/logical/event_decoder.go</a> </details> <details> <summary>logical_replication_writer_processor.go<code>Make CRUD a selectable immediate-mode writer</code> <code>+17/-5</code></summary> <br/> >Make CRUD a selectable immediate-mode writer > ><pre> >• Adds &#x27;crud&#x27; to the writer setting and constructs the CRUD batch handler when selected. Also guards cleanup against a nil handler after interrupted initialization. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-f6e24d38db7bfcad57c57d8f8b95331d9cbf8ca8a5084f743e89545d3de2d157'>pkg/crosscluster/logical/logical_replication_writer_processor.go</a> </details> <details> <summary>sql_crud_writer.go<code>Add the CRUD batch handler</code> <code>+179/-0</code></summary> <br/> >Add the CRUD batch handler > ><pre> >• Implements &#x27;BatchHandler&#x27; by decoding events, optionally discarding deletes, and stably grouping events by destination table. Delegates each group to a table handler and translates its results into batch statistics. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-ecb0ab8d5ccaa2df8e8339ece12e2e0983946e8f9ab847db6d5d7afc54da45e7'>pkg/crosscluster/logical/sql_crud_writer.go</a> </details> <details> <summary>sql_row_reader.go<code>Apply a session override to row refresh queries</code> <code>+6/-2</code></summary> <br/> >Apply a session override to row refresh queries > ><pre> >• Accepts and stores an internal executor override, then uses it when querying local rows during stale-value refresh. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-4962f05bf747f8cfac5c43915728673443741ca7395f9108fa77ff843b1b7634'>pkg/crosscluster/logical/sql_row_reader.go</a> </details> <details> <summary>sql_row_writer.go<code>Apply replication session metadata to CRUD writes</code> <code>+14/-9</code></summary> <br/> >Apply replication session metadata to CRUD writes > ><pre> >• Accepts a session override and adds the replication origin timestamp and origin ID when executing row writes. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-6b3d261201fcd2f42ba2a6a41b04b335261380d8bd4e9880528d1f5859099c4c'>pkg/crosscluster/logical/sql_row_writer.go</a> </details> <details> <summary>table_batch_handler.go<code>Configure per-table CRUD handlers with job context</code> <code>+22/-3</code></summary> <br/> >Configure per-table CRUD handlers with job context > ><pre> >• Propagates the caller context and job ID into handler construction, and gives internal reads and writes a job-specific application name. Adds conversion from table-level conflict and refresh statistics to batch statistics. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-e9aa3b5d11d4c29e3077087f6bd036d3643925539046ed8c4c568d3eb21638cc'>pkg/crosscluster/logical/table_batch_handler.go</a> </details> </dd> </dl> </details> <details> <summary>Bug fix (1) <code> +3 / -3 </code></summary> <dl> <dd> <details> <summary>replication_statements.go<code>Serialize SQL casts with qualified type names</code> <code>+3/-3</code></summary> <br/> >Serialize SQL casts with qualified type names > ><pre> >• Uses statement serialization instead of string formatting before parsing so generated type casts retain fully qualified names. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-be11b86668b7068ddb2d2dc35530cf6d25b46b89297edc9713f6bf92316b7288'>pkg/crosscluster/logical/replication_statements.go</a> </details> </dd> </dl> </details> <details> <summary>Refactor (1) <code> +30 / -12 </code></summary> <dl> <dd> <details> <summary>lww_kv_processor.go<code>Extract shared CDC decoder construction</code> <code>+30/-12</code></summary> <br/> >Extract shared CDC decoder construction > ><pre> >• Moves CDC decoder setup into a helper shared by the legacy KV processor and the new CRUD event decoder. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-62f417b8bfdc74a9102668bda82847dee37eec929f97b4873945f800f63db48a'>pkg/crosscluster/logical/lww_kv_processor.go</a> </details> </dd> </dl> </details> <details> <summary>Tests (5) <code> +9 / -7 </code></summary> <dl> <dd> <details> <summary>logical_replication_job_test.go<code>Run tombstone replication test without an explicit writer mode</code> <code>+2/-2</code></summary> <br/> >Run tombstone replication test without an explicit writer mode > ><pre> >• Removes explicit VALIDATED mode from two replication streams in the tombstone update test, allowing them to use the configured mode. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-14721cc326ec46270739882bd5237c7a80ff8f040d02130cc4a752be46d09945'>pkg/crosscluster/logical/logical_replication_job_test.go</a> </details> <details> <summary>sql_row_reader_test.go<code>Adapt row reader tests to the session override</code> <code>+3/-2</code></summary> <br/> >Adapt row reader tests to the session override > ><pre> >• Passes an empty internal executor override to the updated reader constructor. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-93bcdb71bd5321cb71c91970ff0e65f138b08121ca2eb2dab27baa343c024f3a'>pkg/crosscluster/logical/sql_row_reader_test.go</a> </details> <details> <summary>sql_row_writer_test.go<code>Adapt row writer tests to the session override</code> <code>+2/-1</code></summary> <br/> >Adapt row writer tests to the session override > ><pre> >• Passes an empty internal executor override to the updated writer constructor. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-23db8741be042f133ab618f884002002ef5ba9e0bf2ddb598f24600c28d5e6dc'>pkg/crosscluster/logical/sql_row_writer_test.go</a> </details> <details> <summary>table_batch_handler_test.go<code>Adapt table handler tests to context and job ID</code> <code>+1/-1</code></summary> <br/> >Adapt table handler tests to context and job ID > ><pre> >• Supplies a context and test job ID to the updated table handler constructor. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-54c986ba8f89b412a550528c234779f532860de6b5b434ba19dc25c682326869'>pkg/crosscluster/logical/table_batch_handler_test.go</a> </details> <details> <summary>ldr_statements<code>Update the generated SELECT statement expectation</code> <code>+1/-1</code></summary> <br/> >Update the generated SELECT statement expectation > ><pre> >• Adjusts the expected statement to reflect serialized, qualified type-cast output. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-89d6eb46428cd6a859f5c87598dfca94922e5a45b52e694e9c8ffbd5424490e9'>pkg/crosscluster/logical/testdata/ldr_statements</a> </details> </dd> </dl> </details> <details> <summary>Other (1) <code> +1 / -0 </code></summary> <dl> <dd> <details> <summary>BUILD.bazel<code>Include the CRUD writer in the Bazel library</code> <code>+1/-0</code></summary> <br/> >Include the CRUD writer in the Bazel library > ><pre> >• Adds the new SQL CRUD writer source file to the logical replication Go library. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-956fab78ca74cbde04e82530a01bc83fbec99700598d0ac35ab93083915565b3'>pkg/crosscluster/logical/BUILD.bazel</a> </details> </dd> </dl> </details> </dd> </dl> </details>
PR-level
<h3>Code Review by Qodo</h3> <code>🐞 Bugs (6)</code> <code>📘 Rule violations (0)</code> <code>📜 Skill insights (0)</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <br/> <img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> <details> <summary> 1. Tenant streams fail with the CRUD writer <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>eventDecoder.decodeEvent</i></b> passes the source key directly to a CDC decoder configured with the >system SQL codec, without removing its tenant prefix. For non-system tenant streams, decoding fails >before either the current or previous row can be applied. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/event_decoder.go[110]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-0c9b889c25d696f75fc06cedd49c767700b219f3034eedaf7e6706de6395a5ecR110-R110)</code> > >```diff >+ decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false) >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The producer forwards source keys unchanged, while the existing processors strip their tenant >prefixes before using this system-codec decoder. ></pre> > > <code>[pkg/crosscluster/producer/event_stream.go\[273-284\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/producer/event_stream.go/#L273-L284)</code> > <code>[pkg/crosscluster/logical/lww_kv_processor.go\[101-111\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_kv_processor.go/#L101-L111)</code> > <code>[pkg/keys/sql.go\[244-252\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/keys/sql.go/#L244-L252)</code> > <code>[pkg/crosscluster/logical/lww_row_processor.go\[347-355\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_row_processor.go/#L347-L355)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >The CRUD decoder passes tenant-prefixed stream keys to a decoder using the system SQL codec, which rejects them. >## Fix Focus Areas >- pkg/crosscluster/logical/event_decoder.go[107-131] >## Recommended Fix >Strip the tenant prefix once before decoding, and use the stripped key for both the current and previous values. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/8bfe79ab-12d5-4c05-987d-824e4e819ffb?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/8bfe79ab-12d5-4c05-987d-824e4e819ffb) <hr/> </details> <details> <summary> 2. Tombstone deletes can loop back <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>sqlRowWriter.getExecutorOverride</i></b> sets origin ID 1 for SQL writes, but the CRUD handler&#x27;s tombstone >path uses a separate <b><i>tombstoneUpdater</i></b> constructed with session data whose origin ID remains zero. >When that path writes a replicated tombstone, the producer&#x27;s origin-zero rangefeed filter can admit >the write as a local event. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/sql_row_writer.go[42]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-6b3d261201fcd2f42ba2a6a41b04b335261380d8bd4e9880528d1f5859099c4cR42-R42)</code> > >```diff >+ session.OriginIDForLogicalDataReplication = 1 >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The tombstone operation bypasses the SQL writer override; its deleter takes origin metadata from >separate session data, and the producer filters for origin zero. ></pre> > > <code>[pkg/crosscluster/logical/table_batch_handler.go\[103-141\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/table_batch_handler.go/#L103-L141)</code> > <code>[pkg/crosscluster/logical/tombstone_updater.go\[143-172\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/tombstone_updater.go/#L143-L172)</code> > <code>[pkg/sql/tablewriter.go\[108-116\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/sql/tablewriter.go/#L108-L116)</code> > <code>[pkg/sql/tablewriter.go\[211-220\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/sql/tablewriter.go/#L211-L220)</code> > <code>[pkg/crosscluster/producer/event_stream.go\[161-161\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/producer/event_stream.go/#L161-L161)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >The new CRUD SQL writes carry the replication origin ID, but the handler's KV tombstone writes do not. >## Fix Focus Areas >- pkg/crosscluster/logical/table_batch_handler.go[103-118] >- pkg/crosscluster/logical/sql_row_writer.go[37-43] >## Recommended Fix >Ensure the session data used to construct the CRUD tombstone deleter carries origin ID 1, while retaining the per-event origin timestamp. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/526f7a63-7c5d-4919-9b60-9fda690fbc86?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/526f7a63-7c5d-4919-9b60-9fda690fbc86) <hr/> </details> <details> <summary> 3. Every insert takes a failed transaction first <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>decodeEvent</i></b> runs <b><i>appendDatums</i></b> on <b><i>PrevRow</i></b> even when <b><i>event.PrevValue.RawBytes</i></b> is empty, >producing a non-nil <b><i>prevRow</i></b> from a deleted row with key columns and NULLs elsewhere. Inserts and >deletes without a previous value therefore miss <b><i>attemptBatch</i></b>’s insert and tombstone branches, >attempt an UPDATE or DELETE against that row, and require a read-refresh and second transaction >after the batch fails. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/event_decoder.go[R131-140]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-0c9b889c25d696f75fc06cedd49c767700b219f3034eedaf7e6706de6395a5ecR131-R140)</code> > >```diff >+ decodedPrevRow, err := d.decoder.DecodeKV(ctx, prevKV, cdcevent.PrevRow, event.PrevValue.Timestamp, false) >+ if err != nil { >+ return decodedEvent{}, err >+ } >+ >+ prevRow := make(tree.Datums, 0, len(dstTable.columns)) >+ prevRow, err = appendDatums(prevRow, decodedPrevRow, dstTable.columns) >+ if err != nil { >+ return decodedEvent{}, err >+ } >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >cdcevent returns non-nil datums for a deleted or absent previous value, and <b><i>DatumsNamed</i></b> iterates >over those columns, so the decoder produces a <b><i>prevRow</i></b> with entries rather than nil. <b><i>attemptBatch</i></b> >selects inserts only when <b><i>event.prevRow == nil</i></b> and its tombstone branch only when ><b><i>len(event.prevRow) == 0</i></b>; the existing SQL processor avoids decoding an absent previous value by >checking <b><i>prevValue.RawBytes == nil</i></b>. ></pre> > > <code>[pkg/ccl/changefeedccl/cdcevent/event.go\[651-675\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/ccl/changefeedccl/cdcevent/event.go/#L651-L675)</code> > <code>[pkg/ccl/changefeedccl/cdcevent/event.go\[124-135\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/ccl/changefeedccl/cdcevent/event.go/#L124-L135)</code> > <code>[pkg/crosscluster/logical/table_batch_handler.go\[155-193\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/table_batch_handler.go/#L155-L193)</code> > <code>[pkg/crosscluster/logical/lww_row_processor.go\[365-384\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_row_processor.go/#L365-L384)</code> > <code>[pkg/crosscluster/logical/event_decoder.go\[127-147\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/event_decoder.go/#L127-L147)</code> > <code>[pkg/crosscluster/logical/table_batch_handler.go\[135-184\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/table_batch_handler.go/#L135-L184)</code> > <code>[pkg/ccl/changefeedccl/cdcevent/event.go\[645-664\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/ccl/changefeedccl/cdcevent/event.go/#L645-L664)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >`eventDecoder.decodeEvent` fills `prevRow` when the source event has no previous value, causing the table handler to miss its insert and tombstone branches on the first attempt. > >## Fix Focus Areas >- pkg/crosscluster/logical/event_decoder.go[127-148] > >## Recommended Fix >If `len(event.PrevValue.RawBytes) == 0`, skip decoding the previous row and leave `prevRow` nil. Also leave it nil if the decoded previous row reports `IsDeleted()`; otherwise decode it as today. Add a unit test verifying that an event with no previous value produces `prevRow == nil`. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/a0369cfc-0f03-485e-8959-d4e861a309b8?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/a0369cfc-0f03-485e-8959-d4e861a309b8) <hr/> </details> <br/> <img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> <details> <summary> 4. Failed rows can land in another table&#x27;s DLQ <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>decodeEvent</i></b> updates <b><i>d.lastRow</i></b> only after a successful decode and does not clear it when ><b><i>DecodeKV</i></b> or <b><i>appendDatums</i></b> fails. When a single-event decode fails, <b><i>GetLastRow</i></b> can return the >previous event’s row, which <b><i>maybeDLQ</i></b> and <b><i>dlqClient.Log</i></b> use for the DLQ table, row JSON and >mutation type alongside the failing event’s KV bytes. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/event_decoder.go[R110-125]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-0c9b889c25d696f75fc06cedd49c767700b219f3034eedaf7e6706de6395a5ecR110-R125)</code> > >```diff >+ decodedRow, err := d.decoder.DecodeKV(ctx, event.KeyValue, cdcevent.CurrentRow, event.KeyValue.Value.Timestamp, false) >+ if err != nil { >+ return decodedEvent{}, err >+ } >+ >+ dstTable, ok := d.srcToDest[decodedRow.TableID] >+ if !ok { >+ return decodedEvent{}, errors.AssertionFailedf("table %d not found", decodedRow.TableID) >+ } >+ >+ row := make(tree.Datums, 0, len(dstTable.columns)) >+ row, err = appendDatums(row, decodedRow, dstTable.columns) >+ if err != nil { >+ return decodedEvent{}, err >+ } >+ d.lastRow = decodedRow >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The existing <b><i>sqlRowProcessor</i></b> clears <b><i>lastRow</i></b> on decode failure, but the new decoder returns on >error before assigning the current row and does not clear the previous one. The writer’s >single-event failure path passes <b><i>bh.GetLastRow()</i></b> to <b><i>maybeDLQ</i></b>, and <b><i>deadLetterQueueClient.Log</i></b> >selects the DLQ table using that row’s <b><i>TableID</i></b>. ></pre> > > <code>[pkg/crosscluster/logical/lww_row_processor.go\[355-360\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_row_processor.go/#L355-L360)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[1065-1090\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L1065-L1090)</code> > <code>[pkg/crosscluster/logical/dead_letter_queue.go\[155-190\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/dead_letter_queue.go/#L155-L190)</code> > <code>[pkg/crosscluster/logical/event_decoder.go\[107-125\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/event_decoder.go/#L107-L125)</code> > <code>[pkg/crosscluster/logical/sql_crud_writer.go\[155-157\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/sql_crud_writer.go/#L155-L157)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[1059-1087\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L1059-L1087)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >A failed decode can leave `eventDecoder.lastRow` pointing to an earlier event, causing a single-event DLQ record to use the wrong row or table. > >## Fix Focus Areas >- pkg/crosscluster/logical/event_decoder.go[107-125] > >## Recommended Fix >Clear `d.lastRow` to `cdcevent.Row{}` at the start of each `decodeEvent` attempt. Assign `d.lastRow = decodedRow` immediately after `DecodeKV` succeeds, so an `appendDatums` failure still leaves the current event’s row available. This matches the clearing behavior of `sqlRowProcessor.ProcessRow`. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/57161b77-2bcd-42a6-acd1-983cc9b467c6?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/57161b77-2bcd-42a6-acd1-983cc9b467c6) <hr/> </details> <details> <summary> 5. Writer shutdown leaves descriptor leases held <code>🐞 Bug</code> <code>☼ Reliability</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>sqlCrudWriter.Close</i></b> does nothing, leaving descriptor leases held by table handlers’ tombstone >updaters unreleased. When the processor shuts down or <b><i>setupBatchHandlers</i></b> closes old handlers to >resize the pool before another successful checkpoint, those leases remain held until they expire and >can hold up schema changes on replicated tables. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/sql_crud_writer.go[R151-153]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-ecb0ab8d5ccaa2df8e8339ece12e2e0983946e8f9ab847db6d5d7afc54da45e7R151-R153)</code> > >```diff >+// Close implements BatchHandler. >+func (c *sqlCrudWriter) Close(ctx context.Context) { >+} >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >Tombstone updaters store acquired descriptor leases, and <b><i>tableHandler.ReleaseLeases</i></b> delegates to >their release operation. The processor calls <b><i>Close</i></b> on shutdown and when <b><i>setupBatchHandlers</i></b> >resizes the pool, but the CRUD writer’s <b><i>Close</i></b> omits that operation; otherwise, leases are released >only at checkpoints. The legacy KV processor’s <b><i>Close</i></b> demonstrates the available pattern by calling ><b><i>ReleaseLeases</i></b>. ></pre> > > <code>[pkg/crosscluster/logical/table_batch_handler.go\[257-259\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/table_batch_handler.go/#L257-L259)</code> > <code>[pkg/crosscluster/logical/lww_kv_processor.go\[367-369\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_kv_processor.go/#L367-L369)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[718-720\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L718-L720)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[662-668\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L662-L668)</code> > <code>[pkg/crosscluster/logical/tombstone_updater.go\[143-172\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/tombstone_updater.go/#L143-L172)</code> > <code>[pkg/crosscluster/logical/table_batch_handler.go\[253-255\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/table_batch_handler.go/#L253-L255)</code> > <code>[pkg/crosscluster/logical/sql_crud_writer.go\[151-164\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/sql_crud_writer.go/#L151-L164)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[478-490\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L478-L490)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >`sqlCrudWriter.Close` leaves tombstone-updater descriptor leases held when the processor shuts down or old handlers are closed during pool resizing, including when an error prevents another successful checkpoint. > >## Fix Focus Areas >- pkg/crosscluster/logical/sql_crud_writer.go[151-164] > >## Recommended Fix >Have `sqlCrudWriter.Close` call `c.ReleaseLeases(ctx)`, as `kvRowProcessor.Close` does, so it releases every table handler’s leases. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/c2ce1179-9718-4c5e-8839-82cbd15bd9f4?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/c2ce1179-9718-4c5e-8839-82cbd15bd9f4) <hr/> </details> <br/> <img src="https://img.shields.io/badge/Low-634FD1?style=flat-square" height="20px" alt="Informational"> <details> <summary> 6. Debug failure injection does nothing with the new writer <code>🐞 Bug</code> <code>◔ Observability</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>sqlCrudWriter.SetSyntheticFailurePercent</i></b> is an empty method and does not embed <b><i>failureInjector</i></b>, >so <b><i>HandleBatch</i></b> never returns <b><i>errInjected</i></b>. The streaming testing knob <b><i>FailureRate</i></b> and the >operator debug flush hook (<b><i>RecordFlushStart</i></b>) both stop working whenever the crud writer is >selected, including when the metamorphic setting picks it in tests. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/crosscluster/logical/sql_crud_writer.go[R171-174]](https://github.com/drape-io/crbench-qodo-2026-09/pull/116/files#diff-ecb0ab8d5ccaa2df8e8339ece12e2e0983946e8f9ab847db6d5d7afc54da45e7R171-R174)</code> > >```diff >+// SetSyntheticFailurePercent implements BatchHandler. >+func (c *sqlCrudWriter) SetSyntheticFailurePercent(uint32) { >+ >+} >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >Both the SQL and KV processors support synthetic failures (<b><i>failureInjector</i></b> in the SQL processor, ><b><i>rate</i></b> in the KV processor). The writer processor relies on this for its testing knob and for >debug-driven failure injection. ></pre> > > <code>[pkg/crosscluster/logical/lww_row_processor.go\[289-305\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/lww_row_processor.go/#L289-L305)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[769-774\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L769-L774)</code> > <code>[pkg/crosscluster/logical/logical_replication_writer_processor.go\[823-834\]](https://github.com/drape-io/crbench-qodo-2026-09/blob/618a34ba4f89d91aec5ee6eaf6bc64255a49d175/pkg/crosscluster/logical/logical_replication_writer_processor.go/#L823-L834)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >The crud writer ignores synthetic failure injection. > >## Fix Focus Areas >- pkg/crosscluster/logical/sql_crud_writer.go[31-36] >- pkg/crosscluster/logical/sql_crud_writer.go[80-89] >- pkg/crosscluster/logical/sql_crud_writer.go[171-174] > >## Recommended Fix >Embed `failureInjector` in `sqlCrudWriter` and remove the empty `SetSyntheticFailurePercent`. At the start of `HandleBatch`, call `injectFailure()` and return its error if there is one, as the SQL processor does. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> [Dismiss ↗](https://app.qodo.ai/findings/bcf7bf2d-e58e-46ce-bce3-afa09a80b620?action=dismiss&origin=git) | [View ↗](https://app.qodo.ai/findings/bcf7bf2d-e58e-46ce-bce3-afa09a80b620) <hr/> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-context:start --> <details><summary><strong>Context sources</strong></summary> <div>Review mode: <code>Auto: 🧠 Deep</code>: Substantial new replication logic spans multiple independent data-processing paths.</div> <!-- qodo-context:end --> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:start --> <details> <summary><strong>Tip of the day</strong></summary> <br/> <pre>💡 Did you know, you can keep summaries lean with Findings visible per group, which tucks the rest behind a View link</pre> <a href="https://docs.qodo.ai/tips-and-tricks">More tips ↗</a> | <a href="https://app.qodo.ai/configurations?tab=display-preferences">Customize Qodo ↗</a> | <a href="https://docs.qodo.ai">Qodo docs ↗</a> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:end --> <!-- https://github.com/drape-io/crbench-qodo-2026-09/commit/618a34ba4f89d91aec5ee6eaf6bc64255a49d175 --> <a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>