crbench//

← all cases

Backup manifest EndKey stored without Clone is corrupted by a caller's scratch-buffer reuse

Tier 2 hard go · local · real merged PR from cockroachdb/cockroach #137565

Ground-truth defects

d1 · correctness · critical
maybeDoSizeFlush assigns `lastFile.Span.EndKey = newSpan.Key` without cloning, but newSpan.Key aliases the caller's reused scratch buffer in compactSpanEntry. The next WriteKey overwrites scratch in place, silently mutating the already-recorded EndKey and producing BackupManifest_File entries with overlapping/drifted spans (corrupted backup metadata). Sibling Reset paths in the same file clone correctly; maybeDoSizeFlush was missed.
d2 · correctness · high
sameElidedPrefix(newSpan.Key, lastFile.Span.EndKey, ...) is structurally always true in the contiguous-extend path, since isContiguousSpan requires newSpan.Key == lastFile.Span.EndKey by definition -- comparing ElidedPrefix(X) to itself. The guard therefore cannot fire on the exact path it exists to protect: extending a manifest file across a contiguous span whose elided prefix differs from s.elidedPrefix silently mislabels the entry's key-strip prefix, corrupting restore for ordinary backup shapes (a table's span ending where the next table's begins). Distinct from d1 -- same function, different mechanism. The sibling file_sst_sink.go compares spanPrefix against s.elidedPrefix directly and does not have this bug.
d3 · correctness · medium
`WriteKey` takes a full MVCCKey but guards ordering on the row key alone (`s.prevKey.Compare(key.Key) >= 0`), so two MVCC versions of the same row -- which the underlying `PutRawMVCC` accepts, and which the sibling `FileSSTSink.copyPointKeys` copies without any such guard -- fail an assertion. The file's own `setMidRowForPrevKey` already handles the equal-key case correctly.

The change under review

diff (truncated — full diff)
diff --git a/pkg/backup/backupsink/BUILD.bazel b/pkg/backup/backupsink/BUILD.bazel
index af804ba9f115..a2a69ba5709f 100644
--- a/pkg/backup/backupsink/BUILD.bazel
+++ b/pkg/backup/backupsink/BUILD.bazel
@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "file_sst_sink.go",
         "sink_utils.go",
+        "sst_sink_key_writer.go",
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/backup/backupsink",
     visibility = ["//visibility:public"],
@@ -31,7 +32,10 @@ go_library(
 
 go_test(
     name = "backupsink_test",
-    srcs = ["file_sst_sink_test.go"],
+    srcs = [
+        "file_sst_sink_test.go",
+        "sst_sink_key_writer_test.go",
+    ],
     embed = [":backupsink"],
     deps = [
         "//pkg/backup/backuppb",
@@ -44,6 +48,7 @@ go_test(
         "//pkg/settings/cluster",
         "//pkg/sql/execinfrapb",
         "//pkg/storage",
+        "//pkg/testutils",
         "//pkg/util/encoding",
         "//pkg/util/hlc",
         "//pkg/util/leaktest",
diff --git a/pkg/backup/backupsink/file_sst_sink.go b/pkg/backup/backupsink/file_sst_sink.go
index d9db62e6a133..31ce7e1e0231 100644
--- a/pkg/backup/backupsink/file_sst_sink.go
+++ b/pkg/backup/backupsink/file_sst_sink.go
@@ -37,11 +37,6 @@ var (
 		settings.WithPublic)
 )
 
-type FileSSTSinkWriter interface {
-	io.Closer
-	Flush(context.Context) error
-}
-
 type ExportedSpan struct {
 	Metadata       backuppb.BackupManifest_File
 	DataSST        []byte
@@ -84,7 +79,9 @@ type FileSSTSink struct {
 	// flush. This counter resets on each flush.
 	completedSpans int32
 
-	elidePrefix roachpb.Key
+	// elidedPrefix represents the elided prefix of the last exportSpan/key written to the sink.
+	// This resets on each flush.
+	elidedPrefix roachpb.Key
 
 	// stats contain statistics about the actions of the FileSSTSink over its
 	// entire lifespan.
@@ -98,12 +95,16 @@ type FileSSTSink struct {
 }
 
 // fileSpanByteLimit is the maximum size of a file span that can be extended.
-const fileSpanByteLimit = 64 << 20
+var fileSpanByteLimit int64 = 64 << 20
 
 func MakeFileSSTSink(
 	conf SSTSinkConf, dest cloud.ExternalStorage, pacer *admission.Pacer,
 ) *FileSSTSink {
-	return &FileSSTSink{conf: conf, dest: dest, pacer: pacer}
+	return &FileSSTSink{
+		conf:  conf,
+		dest:  dest,
+		pacer: pacer,
+	}
 }
 
 func (s *FileSSTSink) Write(ctx context.Context, resp ExportedSpan) (roachpb.Key, error) {
@@ -120,12 +121,12 @@ func (s *FileSSTSink) Write(ctx context.Context, resp ExportedSpan) (roachpb.Key
 	// since it overlaps but SSTWriter demands writes in-order.
 	if len(s.flushedFiles) > 0 {
 		last := s.flushedFiles[len(s.flushedFiles)-1].Span.EndKey
-		if span.Key.Compare(last) < 0 || !bytes.Equal(spanPrefix, s.elidePrefix) {
+		if span.Key.Compare(last) < 0 || !bytes.Equal(spanPrefix, s.elidedPrefix) {
 			log.VEventf(ctx, 1, "flushing backup file %s of size %d because span %s cannot append before %s",
 				s.outName, s.flushedSize, span, last,
 			)
 			s.stats.oooFlushes++
-			if err := s.flushFile(ctx); err != nil {
+			if err := s.Flush(ctx); err != nil {
 				return nil, err
 			}
 		}
@@ -137,7 +138,7 @@ func (s *FileSSTSink) Write(ctx context.Context, resp ExportedSpan) (roachpb.Key
 			return nil, err
 		}
 	}
-	s.elidePrefix = append(s.elidePrefix[:0], spanPrefix...)
+	s.elidedPrefix = append(s.elidedPrefix[:0], spanPrefix...)
 
 	log.VEventf(ctx, 2, "writing %s to backup file %s", span, s.outName)
 
@@ -145,7 +146,7 @@ func (s *FileSSTSink) Write(ctx context.Context, resp ExportedSpan) (roachpb.Key
 	// then surface all the range keys and flush.
 	//
 	// TODO(msbutler): investigate using single a single iterator that surfaces
-	// all point keys first and then all range keys
+	// all point keys first and then all range keys.
 	maxKey, err := s.copyPointKeys(ctx, resp.DataSST)
 	if err != nil {
 		return nil, err
@@ -208,7 +209,7 @@ func (s *FileSSTSink) Write(ctx context.Context, resp ExportedSpan) (roachpb.Key
 	if s.flushedSize > targetFileSize.Get(s.conf.Settings) && !s.midRow {
 		s.stats.sizeFlushes++
 		log.VEventf(ctx, 2, "flushing backup file %s with size %d", s.outName, s.flushedSize)
-		if err := s.flushFile(ctx); err != nil {
+		if err := s.Flush(ctx); err != nil {
 			return nil, err
 		}
 	} else {
@@ -238,10 +239,6 @@ func (s *FileSSTSink) Close() error {
 }
 
 func (s *FileSSTSink) Flush(ctx context.Context) error {
-	return s.flushFile(ctx)
-}
-
-func (s *FileSSTSink) flushFile(ctx context.Context) error {
 	if s.out == nil {
 		// If the writer was not initialized but the sink has reported completed
 		// spans then there were empty ExportRequests that were processed by the
@@ -310,7 +307,7 @@ func (s *FileSSTSink) flushFile(ctx context.Context) error {
 	}
 
 	s.flushedFiles = nil
-	s.elidePrefix = s.elidePrefix[:0]
+	s.elidedPrefix = s.elidedPrefix[:0]
 	s.flushedSize = 0
 	s.flushedRevStart.Reset()
 	s.completedSpans = 0
@@ -350,6 +347,7 @@ func (s *FileSSTSink) open(ctx context.Context) error {
 
 	return nil
 }
+
 func (s *FileSSTSink) copyPointKeys(ctx context.Context, dataSST []byte) (roachpb.Key, error) {
 	iterOpts := storage.IterOptions{
 		KeyTypes:   storage.IterKeyTypePointsOnly,
@@ -376,9 +374,9 @@ func (s *FileSSTSink) copyPointKeys(ctx context.Context, dataSST []byte) (roachp
 			break
 		}
 		k := iter.UnsafeKey()
-		suffix, ok := bytes.CutPrefix(k.Key, s.elidePrefix)
+		suffix, ok := bytes.CutPrefix(k.Key, s.elidedPrefix)
 		if !ok {
-			return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", k.Key, s.elidePrefix)
+			return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", k.Key, s.elidedPrefix)
 		}
 		k.Key = suffix
 
@@ -457,11 +455,11 @@ func (s *FileSSTSink) copyRangeKeys(dataSST []byte) (roachpb.Key, error) {
 				maxKey = append(maxKey[:0], rk.EndKey...)
 			}
 			var ok bool
-			if rk.StartKey, ok = bytes.CutPrefix(rk.StartKey, s.elidePrefix); !ok {
-				return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", rk.StartKey, s.elidePrefix)
+			if rk.StartKey, ok = bytes.CutPrefix(rk.StartKey, s.elidedPrefix); !ok {
+				return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", rk.StartKey, s.elidedPrefix)
 			}
-			if rk.EndKey, ok = bytes.CutPrefix(rk.EndKey, s.elidePrefix); !ok {
-				return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", rk.EndKey, s.elidePrefix)
+			if rk.EndKey, ok = bytes.CutPrefix(rk.EndKey, s.elidedPrefix); !ok {
+				return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", rk.EndKey, s.elidedPrefix)
 			}
 			if err := s.sst.PutRawMVCCRangeKey(rk, v.Value); err != nil {
 				return nil, err
diff --git a/pkg/backup/backupsink/file_sst_sink_test.go b/pkg/backup/backupsink/file_sst_sink_test.go
index 042f690229c9..a0964e5561a5 100644
--- a/pkg/backup/backupsink/file_sst_sink_test.go
+++ b/pkg/backup/backupsink/file_sst_sink_test.go
@@ -1,4 +1,4 @@
-// Copyright 2023 The Cockroach Authors.
+// Copyright 2024 The Cockroach Authors.
 //
 // Use of this software is governed by the CockroachDB Software License
 // included in the /LICENSE file.
@@ -93,7 +93,7 @@ func TestFileSSTSinkExtendOneFile(t *testing.T) {
 
 	st := cluster.MakeTestingClusterSettings()
 	targetFileSize.Override(ctx, &st.SV, 20)
-	sink, _ := fileSSTSinkTestSetup(t, st)
+	sink, _ := fileSSTSinkTestSetup(t, st, execinfrapb.ElidePrefix_None)
 
 	resumeKey, err := sink.Write(ctx, exportResponse1)
 	require.NoError(t, err)
@@ -381,11 +381,10 @@ func TestFileSSTSinkWrite(t *testing.T) {
 					targetFileSize.Override(ctx, &st.SV, 10<<10)
 				}
 
-				sink, store := fileSSTSinkTestSetup(t, st)
+				sink, store := fileSSTSinkTestSetup(t, st, elide)
 				defer func() {
 					require.NoError(t, sink.Close())
 				}()
-				sink.conf.ElideMode = elide
 
 				var resumeKey roachpb.Key
 				var err error
@@ -461,6 +460,10 @@ func s2k0(s string) roachpb.Key {
 	return s2kWithColFamily(s, 0)
 }
 
+func s2k1(s string) roachpb.Key {
+	return s2kWithColFamily(s, 1)
+}
+
 // TestFileSSTSinkStats tests the internal counters and stats of the FileSSTSink under
 // different write scenarios.
 func TestFileSSTSinkStats(t *testing.T) {
@@ -472,7 +475,7 @@ func TestFileSSTSinkStats(t *testing.T) {
 	st := cluster.MakeTestingClusterSettings()
 	targetFileSize.Override(ctx, &st.SV, 10<<10)
 
-	sink, _ := fileSSTSinkTestSetup(t, st)
+	sink, _ := fileSSTSinkTestSetup(t, st, execinfrapb.ElidePrefix_None)
 
 	defer func() {
 		require.NoError(t, sink.Close())
@@ -915,6 +918,19 @@ type kvAndTS struct {
 	timestamp int64
 }
 
+func (k kvAndTS) toMVCCKV(enc func(string) roachpb.Key) mvccKV {
+	v := roachpb.Value{}
+	v.SetBytes(k.value)
+	v.InitChecksum(nil)
+	return mvccKV{
+		key: storage.MVCCKey{
+			Key:       enc(k.key),
+			Timestamp: hlc.Timestamp{WallTime: k.timestamp},
+		},
+		value: v.RawBytes,
+	}
+}
+
 type rangeKeyAndTS struct {
 	key       string
 	endKey    string
@@ -1029,23 +1045,26 @@ func (b *exportedSpanBuilder) buildWithEncoding(stringToKey func(string) roachpb
 }
 
 func fileSSTSinkTestSetup(
-	t *testing.T, settings *cluster.Settings,
+	t *testing.T, settings *cluster.Settings, elideMode execinfrapb.ElidePrefix,
 ) (*FileSSTSink, cloud.ExternalStorage) {
+	conf, store := sinkTestSetup(t, settings, elideMode)
+	sink := MakeFileSSTSink(conf, store, nil /* pacer */)
+	return sink, store
+}
 
+func sinkTestSetup(
+	t *testing.T, settings *cluster.Settings, elideMode execinfrapb.ElidePrefix,
+) (SSTSinkConf, cloud.ExternalStorage) {
 	store := nodelocal.TestingMakeNodelocalStorage(t.TempDir(), settings, cloudpb.ExternalStorage{})
-
-	// Never block.
 	progCh := make(chan execinfrapb.RemoteProducerMetadata_BulkProcessorProgress, 100)
-
 	sinkConf := SSTSinkConf{
-		ID:       1,
-		Enc:      nil,
-		ProgCh:   progCh,
-		Settings: &settings.SV,
+		ID:        1,
+		Enc:       nil,
+		ProgCh:    progCh,
+		Settings:  &settings.SV,
+		ElideMode: elideMode,
 	}
-
-	sink := MakeFileSSTSink(sinkConf, store, nil /* pacer */)
-	return sink, store
+	return sinkConf, store
 }
 
 func checkFiles(
@@ -1053,7 +1072,7 @@ func checkFiles(
 	store cloud.ExternalStorage,
 	files []backuppb.BackupManifest_File,
 	expectedFileSpans []roachpb.Spans,
-	elided bool,
+	eliding bool,
 ) error {
 	iterOpts := storage.IterOptions{
 		KeyTypes:   storage.IterKeyTypePointsOnly,
@@ -1101,7 +1120,7 @@ func checkFiles(
 
 			key := iter.UnsafeKey()
 
-			if !endKeyInclusiveSpansContainsKey(spans, key.Key, elided) {
+			if !endKeyInclusiveSpansContainsKey(spans, key.Key, eliding) {
 				return errors.Newf("key %v in file %s not contained by its spans [%v]", key.Key, f, spans)
 			}
 		}
@@ -1111,9 +1130,9 @@ func checkFiles(
 	return nil
 }
 
-func endKeyInclusiveSpansContainsKey(spans roachpb.Spans, key roachpb.Key, elided bool) bool {
+func endKeyInclusiveSpansContainsKey(spans roachpb.Spans, key roachpb.Key, eliding bool) bool {
 	for _, sp := range spans {
-		if elided {
+		if eliding {
 			sp.Key, _ = keys.StripTablePrefix(sp.Key)
 			sp.EndKey, _ = keys.StripTablePrefix(sp.EndKey)
 		}
diff --git a/pkg/backup/backupsink/sink_utils.go b/pkg/backup/backupsink/sink_utils.go
index fb483486fc6b..556955636648 100644
--- a/pkg/backup/backupsink/sink_utils.go
+++ b/pkg/backup/backupsink/sink_utils.go
@@ -6,6 +6,7 @@
 package backupsink
 
 import (
+	"bytes"
 	"fmt"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
@@ -82,3 +83,22 @@ func generateUniqueSSTName(nodeID base.SQLInstanceID) string {
 	return fmt.Sprintf("data/%d.sst",
 		builtins.GenerateUniqueInt(builtins.ProcessUniqueID(nodeID)))
 }
+
+// isContiguousSpan returns true if the first span ends where the second span begins.
+func isContiguousSpan(first, second roachpb.Span) bool {
+	return first.EndKey.Equal(second.Key)
+}
+
+// sameElidedPrefix returns true if the elided prefix of a and b are equal based
+// on the given mode.
+func sameElidedPrefix(a, b roachpb.Key, mode execinfrapb.ElidePrefix) (bool, error) {
+	prefixA, err := ElidedPrefix(a, mode)
+	if err != nil {
+		return false, err
+	}
+	prefixB, err := ElidedPrefix(b, mode)
+	if err != nil {
+		return false, err
+	}
+	return bytes.Equal(prefixA, prefixB), nil
+}
diff --git a/pkg/backup/backupsink/sst_sink_key_writer.go b/pkg/backup/backupsink/sst_sink_key_writer.go
new file mode 100644
index 000000000000..77bbd1314ff1
--- /dev/null
+++ b/pkg/backup/backupsink/sst_sink_key_writer.go
@@ -0,0 +1,276 @@
+// Copyright 2023 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package backupsink
+
+import (
+	"bytes"
+	"context"
+
+	"github.com/cockroachdb/cockroach/pkg/backup/backuppb"
+	"github.com/cockroachdb/cockroach/pkg/cloud"
+	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/sql/execinfrapb"
+	"github.com/cockroachdb/cockroach/pkg/storage"
+	"github.com/cockroachdb/cockroach/pkg/util/admission"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
+	"github.com/cockroachdb/errors"
+)
+
+type SSTSinkKeyWriter struct {
+	FileSSTSink
+	// prevKey represents the last key written using WriteKey. When writing a new
+	// key, this helps determine if the last key written was mid-row. This resets
+	// on each flush.
+	prevKey roachpb.Key
+
+	// Caching the targetFileSize from the cluster settings to avoid multiple
+	// lookups during writes.
+	targetFileSize int64
+}
+
+func MakeSSTSinkKeyWriter(
+	conf SSTSinkConf, dest cloud.ExternalStorage, pacer *admission.Pacer,
+) (*SSTSinkKeyWriter, error) {
+	if conf.ElideMode == execinfrapb.ElidePrefix_None {
+		return nil, errors.New("KeyWriter does not support ElidePrefix_None")
+	}
+	return &SSTSinkKeyWriter{
+		FileSSTSink:    *MakeFileSSTSink(conf, dest, pacer),
+		targetFileSize: targetFileSize.Get(conf.Settings),
+	}, nil
+}
+
+// WriteKey writes a single key to the SST file. The key should be the full key,
+// including the prefix. Reset needs to be called prior to WriteKey whenever
+// writing keys from a new span. Keys must also be written in order.
+//
+// After writing the last key for a span, AssumeNotMidRow must be called to
+// enforce the invariant that BackupManifest_File spans do not end mid-row.
+//
+// Flush should be called before the sink is closed to ensure the SST
+// is written to the destination.
+func (s *SSTSinkKeyWriter) WriteKey(ctx context.Context, key storage.MVCCKey, value []byte) error {
+	if len(s.flushedFiles) == 0 {
+		return errors.AssertionFailedf(
+			"no BackupManifest_File to write key to, call Reset before WriteKey",
+		)
+	}
+	if !s.flushedFiles[len(s.flushedFiles)-1].Span.ContainsKey(key.Key) {
+		return errors.AssertionFailedf(
+			"key %s not in span %s, call Reset on the new span",
+			key.Key,
+			s.flushedFiles[len(s.flushedFiles)-1].Span,
+		)
+	}
+	if s.prevKey != nil && s.prevKey.Compare(key.Key) >= 0 {
+		return errors.AssertionFailedf("key %s must be greater than previous key %s", key.Key, s.prevKey)
+	}
+	// At this point, because of the Reset invariant, we can make the following
+	// assumptions about the new key:
+	// - The new key comes after the previous key.
+	// - This new key belongs to the span of the last BackupManifest_File in
+	//   s.flushedFiles.
+	s.setMidRowForPrevKey(key.Key)
+	if err := s.maybeDoSizeFlush(ctx, key.Key); err != nil {
+		return err
+	}
+
+	elidedKey, err := elideMVCCKey(key, s.conf.ElideMode)
+	if err != nil {
+		return err
+	}
+	// TODO (kev-cao): Should add some method of tracking `Rows` (and potentially
+	// `IndexEntries`).
+	keyAsRowCount := roachpb.RowCount{
+		DataSize: int64(len(key.Key)) + int64(len(value)),
+	}
+
+	if err := s.sst.PutRawMVCC(elidedKey, value); err != nil {
+		return err
+	}
+
+	s.flushedFiles[len(s.flushedFiles)-1].EntryCounts.Add(keyAsRowCount)
+	s.flushedSize += keyAsRowCount.DataSize
+	s.prevKey = append(s.prevKey[:0], key.Key...)
+	// Until the next key is written, we cannot determine if this key is mid-row.
+	// As a safeguard, we assume it is mid-row, and if it is the last key to be
+	// written in this span, the caller has the responsibility of explicitly
+	// marking the key as not mid-row by calling AssumeNotMidRow.
+	s.midRow = true
+	return nil
+}
+
+// AssumeNotMidRow marks the last key written as not mid-row. This exists to
+// ensure that the caller explicitly is aware that SSTSinkKeyWriter is unable
+// to determine if the last key written in a span is mid-row and must enforce
+// the invariant themselves.
+func (s *SSTSinkKeyWriter) AssumeNotMidRow() {
+	s.midRow = false
+}
+
+// Reset resets the SSTSinkKeyWriter to write to a new span. It ensures that an
+// open BackupManifest_File exists for the new span. It will either open a new
+// BackupManifest_File, or if the new span contiguously extends the last span,
+// it will extend it if it remains under the fileSpanByteLimit. The caller is
+// responsible for ensuring that spans that are reset on do not end mid-row
+// and calling AssumeNotMidRow before resetting to enforce this invariant.
+// Any time a new span is being written, Reset MUST be called prior to any
+// WriteKey calls.
+func (s *SSTSinkKeyWriter) Reset(ctx context.Context, newSpan roachpb.Span) error {
+	log.VEventf(ctx, 2, "resetting sink to span %s", newSpan)
+	if s.midRow {
+		return errors.AssertionFailedf("cannot reset after writing a mid-row key")
+	}
+	if err := s.maybeFlushNonExtendableSST(newSpan); err != nil {
+		return err
+	}
+	if s.flushedSize >= s.targetFileSize {
+		if err := s.doSSTSizeFlush(ctx); err != nil {
+			return err
+		}
+	}
+	if s.out == nil {
+		if err := s.open(ctx); err != nil {
+			return err
+		}
+	}
+	// At this point, we can assume the new span is either contiguous with the
+	// last span or comes after the last span (if it exists).
+	if len(s.flushedFiles) > 0 {

Leaderboard for this case

hard · 1 of 9 bots fully caught it (11% solve rate) · 59% avg coverage.

#BotResultCaughtCoverageFindingsFalse alarms
1 cursor solved 3/3 100.0% 4 0
2 Claude Code (Opus 5.5) partial 2/3 42.9% 4 0
3 Drape (Deep Review) partial 2/3 42.9% 7 0
4 Drape (Review) partial 2/3 42.9% 6 0
5 Mjolnir (Opus 5.5) partial 2/3 42.9% 3 0
6 Mjolnir (Sonnet 5.5) partial 2/3 42.9% 4 0
7 Cubic partial 1/3 14.3% 2 0
8 greptile partial 1/3 14.3% 2 0
9 qodo partial 1/3 14.3% 4 0

Bot results

Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✓ exact
d3 medium ✓ off by one
valid · matches d3 · impact medium
valid · matches d2 · impact high
valid · impact medium
WriteKey always sets `s.midRow = true` after a write. When EnsureSafeSplitKey fails, setMidRowForPrevKey returns early and never clears it, so maybeDoSizeFlush hits `if s.midRow { return nil }` and skips the flush. The comment treats this case as not mid-row, and adjustFileEndKey returns false for the same case, so the behaviour is inconsistent. [grounded: tree pkg/backup/backupsink/sst_sink_key_writer.go] [consensus 3/3: valid=3]
valid · impact low
flushedSize is built from the full (unelided) key length plus the value, without the timestamp, while the SST stores the elided key. So the value used for targetFileSize decisions does not match the bytes written. FileSSTSink uses len(DataSST) instead. This is an approximation, so the real impact is small. [grounded: tree pkg/backup/backupsink/sst_sink_key_writer.go] [consensus 3/5: valid=3, uncertain=1, invalid=1]
raw comments (4)
pkg/backup/backupsink/sst_sink_key_writer.go:69
WriteKey compares only `prevKey` (a roachpb.Key) with `key.Key` and rejects anything `>= 0`. Multiple MVCC versions of the same user key share the same roachpb.Key and differ only by timestamp, so revision-history backups fail with an assertion error on the second version of any key. The ordering check should compare full MVCCKeys (key plus timestamp), or allow equal keys when the timestamp is lower.
pkg/backup/backupsink/sst_sink_key_writer.go:174
maybeFlushNonExtendableSST compares the new span's prefix with the previous span's EndKey, but the SST's elided prefix comes from the start key (`s.elidedPrefix`). A table's span usually ends at the next table's prefix, e.g. [/Table/104, /Table/105). In that case a span starting in /Table/105 counts as having the same prefix, so no flush happens, and the span may even be extended because it is contiguous. Keys from two tables then go into one SST, each with a different prefix elided. That gives out-of-order or colliding elided keys and file spans whose elided prefix is wrong on restore. Compare against `s.elidedPrefix` or the last file's Span.Key prefix instead, as FileSSTSink.Write does.
pkg/backup/backupsink/sst_sink_key_writer.go:243
When EnsureSafeSplitKey fails, setMidRowForPrevKey returns without changing `midRow`. The comment says the key is then a range boundary (not mid-row), but WriteKey always sets `midRow = true` after each write, so the flag stays true. Any run of keys that are not family-suffixed SQL keys therefore never allows a size or file-span-limit flush in maybeDoSizeFlush, and files can grow without bound. FileSSTSink handles this case as not mid-row; this should set `s.midRow = false` here too.
pkg/backup/backupsink/sst_sink_key_writer.go:89
Per-key DataSize is computed from `len(key.Key)` plus the value length. The bytes actually written are the elided key plus an MVCC timestamp, so size accounting, and with it the targetFileSize and fileSpanByteLimit flush decisions, does not match the bytes written.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✗ —
d3 medium ✓ off by one
valid · matches d3 · impact medium
minor · impact low
The startTime and endTime fields are set in withRawKVs, but nothing in the new test file reads them, so the claim is accurate. It is a cleanup note about unused test-helper code, not a claim that anything behaves incorrectly. [grounded: diff] [consensus 5/5: minor=5]
raw comments (3)
pkg/backup/backupsink/sst_sink_key_writer.go:69
<!-- cubic:v=b9414571-1c16-47d8-9915-17f91de60fda --> <!-- cubic:review-run=ad085615-03d0-5501-b629-d51137ff4277 --> <!-- metadata:{"confidence":9} --> P1: When a backup contains multiple revisions of the same user key, `WriteKey` rejects the second revision because it compares only `key.Key`. Compare full `storage.MVCCKey` ordering while retaining the raw key separately for row-boundary detection. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. At pkg/backup/backupsink/sst_sink_key_writer.go, line 69: <comment>When a backup contains multiple revisions of the same user key, `WriteKey` rejects the second revision because it compares only `key.Key`. Compare full `storage.MVCCKey` ordering while retaining the raw key separately for row-boundary detection.</comment> <file context> @@ -0,0 +1,276 @@ + s.flushedFiles[len(s.flushedFiles)-1].Span, + ) + } + if s.prevKey != nil && s.prevKey.Compare(key.Key) >= 0 { + return errors.AssertionFailedf("key %s must be greater than previous key %s", key.Key, s.prevKey) + } </file context> ``` </details>
pkg/backup/backupsink/sst_sink_key_writer_test.go:544
<!-- cubic:v=326e7d28-9661-46f8-b87b-df259b19b14a --> <!-- cubic:review-run=ad085615-03d0-5501-b629-d51137ff4277 --> <!-- metadata:{"confidence":8} --> P3: `mvccKVSet.startTime` and `endTime` are computed in `withRawKVs` but never read by any test, so the min/max timestamp loop is dead work. Remove the two fields and the `minTS`/`maxTS` bookkeeping unless a later test is meant to assert on them. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. At pkg/backup/backupsink/sst_sink_key_writer_test.go, line 544: <comment>`mvccKVSet.startTime` and `endTime` are computed in `withRawKVs` but never read by any test, so the min/max timestamp loop is dead work. Remove the two fields and the `minTS`/`maxTS` bookkeeping unless a later test is meant to assert on them.</comment> <file context> @@ -0,0 +1,558 @@ + return b.withRawKVs(rawKVs) +} + +func (b *mvccKVSet) withRawKVs(kvs []mvccKV) *mvccKVSet { + b.kvs = kvs + var minTS, maxTS hlc.Timestamp </file context> ``` </details>
PR-level
<!-- cubic:review-summary:start --> **2 issues found** across 6 files <!-- cubic:review-summary:end --> <details> <summary>Prompt for AI agents (unresolved issues)</summary> ```text Check if these issues are valid — if so, understand the root cause of each and fix them. If appropriate, use sub-agents to investigate and fix each issue separately. <file name="pkg/backup/backupsink/sst_sink_key_writer.go"> <violation number="1" location="pkg/backup/backupsink/sst_sink_key_writer.go:69"> P1: When a backup contains multiple revisions of the same user key, `WriteKey` rejects the second revision because it compares only `key.Key`. Compare full `storage.MVCCKey` ordering while retaining the raw key separately for row-boundary detection.</violation> </file> <file name="pkg/backup/backupsink/sst_sink_key_writer_test.go"> <violation number="1" location="pkg/backup/backupsink/sst_sink_key_writer_test.go:544"> P3: `mvccKVSet.startTime` and `endTime` are computed in `withRawKVs` but never read by any test, so the min/max timestamp loop is dead work. Remove the two fields and the `minTS`/`maxTS` bookkeeping unless a later test is meant to assert on them.</violation> </file> ``` </details> <sub>Reply with feedback, questions, or to request a fix.<br /><br />[Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/26/ai_pr_review_github_delivery_128922f9c730340da06e20ce8f2c9e515c63cd0184bc7590b1197720eabb89eb?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F26)</sub> <!-- cubic:review-post:ai_pr_review_github_delivery_128922f9c730340da06e20ce8f2c9e515c63cd0184bc7590b1197720eabb89eb:3474417dbb029ef1a3005bc6c62a6c8892598649:ad085615-03d0-5501-b629-d51137ff4277 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✓ off by one
d2 high ✓ exact
d3 medium ✓ off by one
valid · matches d2 · impact high
valid · matches d1 · impact critical
valid · matches d3 · impact medium
valid · impact medium
WriteKey always sets `s.midRow = true` after a write. On the next write, if `EnsureSafeSplitKey` fails, `setMidRowForPrevKey` returns early without clearing `midRow`. `maybeDoSizeFlush` then sees `midRow` still true and skips both the size flush and the span split. This contradicts the function's own comment ("must be ... a range boundary") and `adjustFileEndKey`, which returns false (not mid-row) on the same error. The bug is in code newly added by this change. [grounded: diff] [consensus 5/5: valid=5]
raw comments (5)
pkg/backup/backupsink/sst_sink_key_writer.go:183
### Prefix check uses span end key **High Severity** <!-- DESCRIPTION START --> `maybeFlushNonExtendableSST` compares elided prefixes using the last file's `EndKey` rather than its start key (or `elidedPrefix`). For contiguous table spans like `[TablePrefix(n), TablePrefix(n+1))` then `[TablePrefix(n+1), TablePrefix(n+2))`, the end key is the next table prefix, so the prefixes compare equal and the SST is not flushed. `Reset` then extends the manifest span across tables and later keys are elided with different prefixes, which can fail `PutRawMVCC` or restore keys into the wrong table. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 8b06ddb8-54d3-4d25-bef6-1766f64f8b31 --> <!-- LOCATIONS START pkg/backup/backupsink/sst_sink_key_writer.go#L178-L183 LOCATIONS END --> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmNhYmQxNmEyLTQ2NjQtNDBhZi1iNmNmLTZjMDUyM2QyN2IyMyIsImVuY3J5cHRpb25LZXkiOiJvVHVTS3d1anhtaVZ6Ti11eXJnZmdFa1l1c3R5US1GS3ZkSE4zdURQUlBvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OmNhYmQxNmEyLTQ2NjQtNDBhZi1iNmNmLTZjMDUyM2QyN2IyMyIsImVuY3J5cHRpb25LZXkiOiJvVHVTS3d1anhtaVZ6Ti11eXJnZmdFa1l1c3R5US1GS3ZkSE4zdURQUlBvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI3LCJjb21taXRTaGEiOiI1OGQwOGMwOTc5NWZjNjQwZTBkYjQ1NTgzNmJmZGU0OGU2M2M1NGU5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 58d08c09795fc640e0db455836bfde48e63c54e9. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
pkg/backup/backupsink/sst_sink_key_writer.go:226
### Size flush aliases caller key bytes **High Severity** <!-- DESCRIPTION START --> `maybeDoSizeFlush` stores `nextKey` directly as a truncated file `EndKey` and as the `Key` of the span passed to `Reset`, without cloning. `WriteKey` copies `prevKey`, but these span fields keep the caller's buffer. Iterator `UnsafeKey` reuse will mutate already-recorded `BackupManifest_File` spans after a size split. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 4c9b80f2-af09-4242-a787-bd3052e388a5 --> <!-- LOCATIONS START pkg/backup/backupsink/sst_sink_key_writer.go#L212-L226 LOCATIONS END --> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjNmMzk0YjM2LTFiZjYtNDZmZi1hOGJkLWZhMGEyZDc2NzdlMSIsImVuY3J5cHRpb25LZXkiOiJFaTZTeEI3NlpQUlNZYnc2TWRJQzYxaDhCay02TW0zVmpDcnVtbnRPLWNFIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjNmMzk0YjM2LTFiZjYtNDZmZi1hOGJkLWZhMGEyZDc2NzdlMSIsImVuY3J5cHRpb25LZXkiOiJFaTZTeEI3NlpQUlNZYnc2TWRJQzYxaDhCay02TW0zVmpDcnVtbnRPLWNFIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI3LCJjb21taXRTaGEiOiI1OGQwOGMwOTc5NWZjNjQwZTBkYjQ1NTgzNmJmZGU0OGU2M2M1NGU5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 58d08c09795fc640e0db455836bfde48e63c54e9. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
pkg/backup/backupsink/sst_sink_key_writer.go:70
### WriteKey rejects MVCC version history **High Severity** <!-- DESCRIPTION START --> `WriteKey` requires each user key to be strictly greater than `prevKey` and ignores timestamps. Valid SST order allows the same key at descending MVCC timestamps. A second version of a key hits the assertion, so revision-history data and compacted incrementals that emit multiple versions cannot be written. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 0cfce09a-097b-488d-85f6-0d890ddbd381 --> <!-- LOCATIONS START pkg/backup/backupsink/sst_sink_key_writer.go#L68-L70 LOCATIONS END --> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjQzMmZlMzhiLWU0ZmItNDYwMC1iZjgxLWQ0ZjViZTA3MDRkNSIsImVuY3J5cHRpb25LZXkiOiJMYmRSMnViTzlCaGxYZVQtNU5FZ29OREI3TllaNmF6VnZscWV3dW1mcnA4IiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjQzMmZlMzhiLWU0ZmItNDYwMC1iZjgxLWQ0ZjViZTA3MDRkNSIsImVuY3J5cHRpb25LZXkiOiJMYmRSMnViTzlCaGxYZVQtNU5FZ29OREI3TllaNmF6VnZscWV3dW1mcnA4IiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI3LCJjb21taXRTaGEiOiI1OGQwOGMwOTc5NWZjNjQwZTBkYjQ1NTgzNmJmZGU0OGU2M2M1NGU5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 58d08c09795fc640e0db455836bfde48e63c54e9. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
pkg/backup/backupsink/sst_sink_key_writer.go:249
### Mid-row flag stuck on parse error **Medium Severity** <!-- DESCRIPTION START --> `setMidRowForPrevKey` returns without changing `midRow` when `EnsureSafeSplitKey` fails. `WriteKey` leaves `midRow` true after every write, so a following non-family key keeps `midRow` true and `maybeDoSizeFlush` skips size and span splits. `adjustFileEndKey` treats that parse failure as not mid-row. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 2f110224-d07b-42e6-ad72-7a34a24df142 --> <!-- LOCATIONS START pkg/backup/backupsink/sst_sink_key_writer.go#L244-L249 LOCATIONS END --> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjVhMDlmYTk4LTYwYzQtNDdlZi05MjdhLWY3ZGQzYTA2NTYzZSIsImVuY3J5cHRpb25LZXkiOiJ2VDg4YVZ3dTRkX3NDWkpic21GTmxGRERpdFJrVlJVZGF4TVdEZHVfRTVvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjVhMDlmYTk4LTYwYzQtNDdlZi05MjdhLWY3ZGQzYTA2NTYzZSIsImVuY3J5cHRpb25LZXkiOiJ2VDg4YVZ3dTRkX3NDWkpic21GTmxGRERpdFJrVlJVZGF4TVdEZHVfRTVvIiwiYnJhbmNoIjoidDItY29ja3JvYWNoLTEzNzU2NSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjI3LCJjb21taXRTaGEiOiI1OGQwOGMwOTc5NWZjNjQwZTBkYjQ1NTgzNmJmZGU0OGU2M2M1NGU5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 58d08c09795fc640e0db455836bfde48e63c54e9. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
PR-level
<!-- BUGBOT_REVIEW --> Cursor Bugbot has reviewed your changes using high effort and found 4 potential issues. <!-- BUGBOT_FIX_ALL --> <a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aToyZjA5ZGYwYy1iNzdmLTQxNGItOGFjZC0yODU2MWJlYzhhYmYiLCJlbmNyeXB0aW9uS2V5IjoiZ3ozdEJoUkpFa0xERFNCRU5xMjNLcXVBYU10VWdJbm11VUhwNXBXZjFuNCIsImJyYW5jaCI6InQyLWNvY2tyb2FjaC0xMzc1NjUiLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <!-- /BUGBOT_FIX_ALL --> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN --> <sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END --> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 58d08c09795fc640e0db455836bfde48e63c54e9. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✓ exact
d3 medium ✓ exact
valid · impact medium
WriteKey strips each key's own prefix via elideMVCCKey and never checks it against s.elidedPrefix, which Reset sets from the span start key. Unlike copyPointKeys, it has no prefix-mismatch assertion, so a key from another table that falls inside the span is written silently with the wrong elision. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
WriteKey always sets s.midRow = true. When EnsureSafeSplitKey errors, setMidRowForPrevKey returns early without clearing it, so maybeDoSizeFlush returns early and size flushes are skipped. adjustFileEndKey treats the same error as not mid-row. [grounded: diff] [consensus 3/3: valid=3]
valid · matches d3 · impact medium
valid · impact low
maybeFlushNonExtendableSST flushes with the stored s.ctx rather than Reset's ctx, so a context passed to a later call is ignored for the blocking ProgCh send. s.ctx is derived from the first caller's context, so the impact is limited. [grounded: diff] [consensus 3/3: valid=3]
valid · matches d2 · impact high
uncertain
Reset does create files without time bounds, but zero time bounds are the normal case in FileSSTSink for non-introduced spans and the new writer has no callers yet. The claimed misreading in backup_job.go cannot be verified from the shown code. [uncertain: no evidence quoted] [consensus 3/3: uncertain=3]
uncertain
The split boundary is the raw next key, and the tests expect s2k0 boundaries. It is unclear whether a family-suffixed span boundary causes real harm, because no key lies between the row key and its family-0 key and downstream splits may normalize it. [uncertain: no evidence quoted] [consensus 3/3: uncertain=3]
raw comments (7)
pkg/backup/backupsink/sst_sink_key_writer.go:82
**Keys elided with their own prefix, not file's** `WriteKey` strips each key's own prefix through `elideMVCCKey` and never checks that it equals `s.elidedPrefix`. A key whose table or tenant differs from the span's start key is then written into the file without any error. Cut `s.elidedPrefix` from the key and return an assertion error when it doesn't match. ``` pkg/backup/backupsink/sst_sink_key_writer.go:82 elidedKey, err := elideMVCCKey(key, s.conf.ElideMode) pkg/backup/backupsink/sst_sink_key_writer.go:83 if err != nil { pkg/backup/backupsink/sst_sink_key_writer.go:84 return err pkg/backup/backupsink/sst_sink_key_writer.go:85 } pkg/backup/backupsink/file_sst_sink.go:376 k := iter.UnsafeKey() pkg/backup/backupsink/file_sst_sink.go:377 suffix, ok := bytes.CutPrefix(k.Key, s.elidedPrefix) pkg/backup/backupsink/file_sst_sink.go:378 if !ok { pkg/backup/backupsink/file_sst_sink.go:379 return nil, errors.AssertionFailedf("prefix mismatch %q does not have %q", k.Key, s.elidedPrefix) pkg/backup/backupsink/file_sst_sink.go:380 } pkg/backup/backupsink/file_sst_sink.go:381 k.Key = suffix ```
pkg/backup/backupsink/sst_sink_key_writer.go:246
**Malformed-suffix keys block all size flushes** When EnsureSafeSplitKey rejects the next key (a table key with an invalid family suffix), setMidRowForPrevKey returns early. midRow then stays true from the previous WriteKey, so maybeDoSizeFlush never flushes or splits by size. In that error branch, set s.midRow = false, as adjustFileEndKey does. ``` pkg/backup/backupsink/sst_sink_key_writer.go:241 func (s *SSTSinkKeyWriter) setMidRowForPrevKey(endKey roachpb.Key) { pkg/backup/backupsink/sst_sink_key_writer.go:242 if s.prevKey == nil { pkg/backup/backupsink/sst_sink_key_writer.go:243 return pkg/backup/backupsink/sst_sink_key_writer.go:244 } pkg/backup/backupsink/sink_utils.go:51 endRowKey, err := keys.EnsureSafeSplitKey(endKey) pkg/backup/backupsink/sink_utils.go:52 if err != nil { pkg/backup/backupsink/sink_utils.go:53 // If the key does not parse a family key, it must be from reaching the end pkg/backup/backupsink/sink_utils.go:54 // of a range and be a range boundary. pkg/backup/backupsink/sink_utils.go:55 return endKey, false pkg/backup/backupsink/sink_utils.go:56 } ```
pkg/backup/backupsink/sst_sink_key_writer.go:68
**Multiple MVCC versions of one key rejected** The ordering check compares only `key.Key`, so writing a second revision of the same key (same key, older timestamp) fails as out of order. Revision-history data always has this shape. Track the previous full MVCCKey and compare with MVCC ordering; treat same-key versions as mid-row. ``` pkg/backup/backupsink/sst_sink_key_writer.go:68 } pkg/backup/backupsink/sst_sink_key_writer.go:69 if s.prevKey != nil && s.prevKey.Compare(key.Key) >= 0 { pkg/backup/backupsink/sst_sink_key_writer.go:70 return errors.AssertionFailedf("key %s must be greater than previous key %s", key.Key, s.prevKey) pkg/backup/backupsink/sst_sink_key_writer.go:96 s.flushedFiles[len(s.flushedFiles)-1].EntryCounts.Add(keyAsRowCount) ```
pkg/backup/backupsink/sst_sink_key_writer.go:190
**Flush ignores the caller's context** `maybeFlushNonExtendableSST` logs and calls `Flush` with `s.ctx`, the context saved at the first open, rather than the `ctx` passed to `Reset`. Cancelling the caller's context then can't interrupt the blocking send on `ProgCh`. Pass `ctx` from `Reset` into `maybeFlushNonExtendableSST`. ``` pkg/backup/backupsink/sst_sink_key_writer.go:174 func (s *SSTSinkKeyWriter) maybeFlushNonExtendableSST(newSpan roachpb.Span) error { pkg/backup/backupsink/sst_sink_key_writer.go:175 if len(s.flushedFiles) == 0 { pkg/backup/backupsink/sst_sink_key_writer.go:176 return nil pkg/backup/backupsink/sst_sink_key_writer.go:177 } pkg/backup/backupsink/file_sst_sink.go:303 select { pkg/backup/backupsink/file_sst_sink.go:304 case <-ctx.Done(): pkg/backup/backupsink/file_sst_sink.go:305 return ctx.Err() pkg/backup/backupsink/file_sst_sink.go:306 case s.conf.ProgCh <- prog: pkg/backup/backupsink/file_sst_sink.go:307 } pkg/backup/backupsink/file_sst_sink.go:320 if s.ctx == nil { ```
pkg/backup/backupsink/sst_sink_key_writer.go:180
**Contiguous span into next table shares SST file** `maybeFlushNonExtendableSST` checks the new span's prefix against the last span's EndKey, not the open file's prefix. A table-5 span ending at `/Table/6` thus matches a span starting at `/Table/6`, and the file is extended instead of flushed. Compare `ElidedPrefix(newSpan.Key)` with `s.elidedPrefix`, as `FileSSTSink.Write` does. ``` pkg/backup/backupsink/sst_sink_key_writer.go:178 lastFile := s.flushedFiles[len(s.flushedFiles)-1] pkg/backup/backupsink/sst_sink_key_writer.go:179 samePrefix, err := sameElidedPrefix(newSpan.Key, lastFile.Span.EndKey, s.conf.ElideMode) pkg/backup/backupsink/sst_sink_key_writer.go:180 if err != nil { pkg/backup/backupsink/sst_sink_key_writer.go:181 return err pkg/backup/backupsink/file_sst_sink.go:122 if len(s.flushedFiles) > 0 { pkg/backup/backupsink/file_sst_sink.go:123 last := s.flushedFiles[len(s.flushedFiles)-1].Span.EndKey pkg/backup/backupsink/file_sst_sink.go:124 if span.Key.Compare(last) < 0 || !bytes.Equal(spanPrefix, s.elidedPrefix) { pkg/backup/backupsink/file_sst_sink.go:125 log.VEventf(ctx, 1, "flushing backup file %s of size %d because span %s cannot append before %s", pkg/backup/backupsink/file_sst_sink.go:126 s.outName, s.flushedSize, span, last, pkg/backup/backupsink/file_sst_sink.go:127 ) ```
pkg/backup/backupsink/sst_sink_key_writer.go:160
**Key-writer files have zero start/end times** Files created in Reset never get StartTime/EndTime, unlike FileSSTSink.Write, which copies resp.Metadata. Every flushed manifest file therefore has zero time bounds, and readers such as the introduced-span classification in backup_job.go treat the file as a plain completed span. Pass the time bounds into Reset and set them on the file. ``` pkg/backup/backupsink/sst_sink_key_writer.go:141 // At this point, we can assume the new span is either contiguous with the pkg/backup/backupsink/sst_sink_key_writer.go:142 // last span or comes after the last span (if it exists). pkg/backup/backupsink/sst_sink_key_writer.go:143 if len(s.flushedFiles) > 0 { pkg/backup/backupsink/sst_sink_key_writer.go:144 lastFile := &s.flushedFiles[len(s.flushedFiles)-1] pkg/backup/backupsink/file_sst_sink.go:168 } else if len(s.flushedFiles) > 0 { pkg/backup/backupsink/file_sst_sink.go:169 last := s.flushedFiles[len(s.flushedFiles)-1] pkg/backup/backupsink/file_sst_sink.go:170 extend = last.Span.EndKey.Equal(span.Key) && pkg/backup/backupsink/file_sst_sink.go:171 last.EndTime == resp.Metadata.EndTime && pkg/backup/backupsink/file_sst_sink.go:172 last.StartTime == resp.Metadata.StartTime && pkg/backup/backupsink/file_sst_sink.go:173 last.EntryCounts.DataSize < fileSpanByteLimit ```
pkg/backup/backupsink/sst_sink_key_writer.go:218
**Size-split boundary is family key, not row key** When `maybeDoSizeFlush` splits a file, it ends the old span and starts the new one at the raw next key, which may carry a column-family suffix. `adjustFileEndKey` instead trims the boundary to the row key so it is a safe split point. Use `keys.EnsureSafeSplitKey(nextKey)` as the boundary when it parses. ``` pkg/backup/backupsink/sst_sink_key_writer.go:212 } pkg/backup/backupsink/sst_sink_key_writer.go:213 newSpan := roachpb.Span{ pkg/backup/backupsink/sst_sink_key_writer.go:214 Key: nextKey, pkg/backup/backupsink/sst_sink_key_writer.go:215 EndKey: lastFile.Span.EndKey, pkg/backup/backupsink/sink_utils.go:70 // If the file end does parse as a family key but the truncated 'row' key is pkg/backup/backupsink/sink_utils.go:71 // still above any key in the file, the end key likely came from export's pkg/backup/backupsink/sink_utils.go:72 // iteration stopping early and setting the end to the resume key, i.e. the pkg/backup/backupsink/sink_utils.go:73 // next real family key. In this case, we are not mid-row, but want to adjust pkg/backup/backupsink/sink_utils.go:74 // our span end -- and where we resume the next file -- to be this row key. pkg/backup/backupsink/sink_utils.go:75 // Thus return the truncated row key and false. ```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✓ exact
d3 medium ✓ exact
valid · impact low
The new maybeFlushNonExtendableSST ignores the ctx passed to Reset. It logs and flushes with s.ctx, which open created from the first caller's ctx. Cancelling the current caller's ctx therefore won't interrupt the blocking ProgCh send in Flush. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
WriteKey always sets midRow=true. If EnsureSafeSplitKey fails, setMidRowForPrevKey returns without resetting it, so maybeDoSizeFlush returns early on `if s.midRow`. adjustFileEndKey treats the same case as not mid-row, so the two disagree. [grounded: diff] [consensus 3/3: valid=3]
valid · matches d2 · impact high
valid · impact low
The setting is read once at construction, so later setting changes don't apply to this sink, unlike FileSSTSink.Write, which reads it per call. The caching is deliberate, so the impact is small. [grounded: diff] [consensus 3/3: valid=3]
valid · impact medium
In TenantAndTable mode, ElidedPrefix on the previous span's EndKey calls StripTablePrefix. That errors when the end key has no valid table encoding after the tenant prefix, for example a tenant end key, /Max, or some PrefixEnd keys, so Reset fails. FileSSTSink.Write only computes the prefix of the new span's start key and compares it against s.elidedPrefix. [grounded: diff] [consensus 3/4: valid=3, uncertain=1]
valid · matches d3 · impact medium
raw comments (6)
pkg/backup/backupsink/sst_sink_key_writer.go:181
**Flush uses stale context captured at first open** `maybeFlushNonExtendableSST` flushes with `s.ctx`, which is the cancellable context created at first `open`. It does not use the caller's `ctx` passed to `Reset`. It also logs with `s.ctx`. Pass `ctx` into `maybeFlushNonExtendableSST` and use it for the log and `Flush`. ``` pkg/backup/backupsink/sst_sink_key_writer.go:174 func (s *SSTSinkKeyWriter) maybeFlushNonExtendableSST(newSpan roachpb.Span) error { pkg/backup/backupsink/sst_sink_key_writer.go:175 if len(s.flushedFiles) == 0 { pkg/backup/backupsink/sst_sink_key_writer.go:176 return nil pkg/backup/backupsink/sst_sink_key_writer.go:177 } pkg/backup/backupsink/file_sst_sink.go:318 func (s *FileSSTSink) open(ctx context.Context) error { pkg/backup/backupsink/file_sst_sink.go:319 s.outName = generateUniqueSSTName(s.conf.ID) pkg/backup/backupsink/file_sst_sink.go:320 if s.ctx == nil { pkg/backup/backupsink/file_sst_sink.go:321 s.ctx, s.cancel = context.WithCancel(ctx) pkg/backup/backupsink/file_sst_sink.go:322 } ```
pkg/backup/backupsink/sst_sink_key_writer.go:243
**Size flushes skipped after unparseable family key** When EnsureSafeSplitKey fails on the next key, setMidRowForPrevKey returns without clearing midRow, which WriteKey left true. maybeDoSizeFlush then skips size and span-limit flushes for that key. Set s.midRow = false in the error branch, as adjustFileEndKey does. ``` pkg/backup/backupsink/sst_sink_key_writer.go:236 } pkg/backup/backupsink/sst_sink_key_writer.go:238 // setMidRowForPrevKey checks if the last key written using WriteKey was mid-row pkg/backup/backupsink/sst_sink_key_writer.go:239 // by using the next key to be written. If no key was previously written using pkg/backup/backupsink/sst_sink_key_writer.go:240 // WriteKey, this is a no-op. pkg/backup/backupsink/sink_utils.go:51 endRowKey, err := keys.EnsureSafeSplitKey(endKey) pkg/backup/backupsink/sink_utils.go:52 if err != nil { pkg/backup/backupsink/sink_utils.go:53 // If the key does not parse a family key, it must be from reaching the end pkg/backup/backupsink/sink_utils.go:54 // of a range and be a range boundary. pkg/backup/backupsink/sink_utils.go:55 return endKey, false pkg/backup/backupsink/sink_utils.go:56 } ```
pkg/backup/backupsink/sst_sink_key_writer.go:168
**Contiguous spans from different tables share one file** `maybeFlushNonExtendableSST` compares the new span's prefix with that of the last span's `EndKey`, which is the same key when spans are contiguous, so they always match. `Reset` then extends the file across a table boundary. The old `Write` compared against `s.elidedPrefix`. Compare `newSpan.Key`'s prefix to `s.elidedPrefix`. ``` pkg/backup/backupsink/sst_sink_key_writer.go:166 s.prevKey = nil pkg/backup/backupsink/sst_sink_key_writer.go:167 return nil pkg/backup/backupsink/sst_sink_key_writer.go:168 } pkg/backup/backupsink/sst_sink_key_writer.go:170 // maybeFlushNonExtendableSST checks if the new span's startKey precedes the pkg/backup/backupsink/file_sst_sink.go:122 if len(s.flushedFiles) > 0 { pkg/backup/backupsink/file_sst_sink.go:123 last := s.flushedFiles[len(s.flushedFiles)-1].Span.EndKey pkg/backup/backupsink/file_sst_sink.go:124 if span.Key.Compare(last) < 0 || !bytes.Equal(spanPrefix, s.elidedPrefix) { pkg/backup/backupsink/file_sst_sink.go:125 log.VEventf(ctx, 1, "flushing backup file %s of size %d because span %s cannot append before %s", ```
pkg/backup/backupsink/sst_sink_key_writer.go:36
**Target file size cached at construction** `targetFileSize` is read once in `MakeSSTSinkKeyWriter`. `FileSSTSink.Write` reads the cluster setting on each call, so later setting changes are ignored for this sink's lifetime. Call targetFileSize.Get(s.conf.Settings) where the size is checked. ``` pkg/backup/backupsink/sst_sink_key_writer.go:36 conf SSTSinkConf, dest cloud.ExternalStorage, pacer *admission.Pacer, pkg/backup/backupsink/sst_sink_key_writer.go:37 ) (*SSTSinkKeyWriter, error) { pkg/backup/backupsink/sst_sink_key_writer.go:38 if conf.ElideMode == execinfrapb.ElidePrefix_None { pkg/backup/backupsink/sst_sink_key_writer.go:39 return nil, errors.New("KeyWriter does not support ElidePrefix_None") ```
pkg/backup/backupsink/sst_sink_key_writer.go:170
**Reset errors on end keys lacking a table** `maybeFlushNonExtendableSST` calls `ElidedPrefix` on the previous span's `EndKey`. In `TenantAndTable` mode that errors for end keys with no table part. The old `Write` only computed the prefix of the new span's start key. Compare the new span's prefix against `s.elidedPrefix`, as `FileSSTSink.Write` does. ``` pkg/backup/backupsink/sst_sink_key_writer.go:166 s.prevKey = nil pkg/backup/backupsink/sst_sink_key_writer.go:167 return nil pkg/backup/backupsink/sst_sink_key_writer.go:168 } pkg/backup/backupsink/sst_sink_key_writer.go:170 // maybeFlushNonExtendableSST checks if the new span's startKey precedes the pkg/keys/sql.go:75 func StripTablePrefix(key roachpb.Key) ([]byte, error) { pkg/keys/sql.go:76 var err error pkg/keys/sql.go:77 key, err = StripTenantPrefix(key) pkg/keys/sql.go:78 if err != nil { pkg/keys/sql.go:79 return nil, err pkg/keys/sql.go:80 } ```
pkg/backup/backupsink/sst_sink_key_writer.go:57
**Multiple versions of one key rejected** WriteKey requires each key to be strictly greater than prevKey, comparing only key.Key and ignoring the timestamp. Revisions of one user key are written as consecutive MVCCKeys, so the second revision fails the check. Compare the full MVCCKey (key, then timestamp descending). ``` pkg/backup/backupsink/sst_sink_key_writer.go:57 if len(s.flushedFiles) == 0 { pkg/backup/backupsink/sst_sink_key_writer.go:58 return errors.AssertionFailedf( pkg/backup/backupsink/sst_sink_key_writer.go:59 "no BackupManifest_File to write key to, call Reset before WriteKey", ```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✗ —
d3 medium ✓ off by one
valid · matches d3 · impact medium
valid · impact low
The new SSTSinkKeyWriter reads the cluster setting once in its constructor and uses the cached `s.targetFileSize` in Reset and maybeDoSizeFlush. FileSSTSink.Write calls `targetFileSize.Get(s.conf.Settings)` on every write. So a change to the setting made while a writer is alive is ignored. The code comment shows the caching was deliberate, but that does not change the behavior. The writers likely live only for a short time, so the impact is low. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
pkg/backup/backupsink/sst_sink_key_writer.go:70
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Valid MVCC versions rejected** When a stream contains multiple timestamped versions of the same logical key, this check compares only the user-key bytes and rejects the second version as out of order. MVCC ordering permits equal user keys ordered by descending timestamp, so this prevents valid revision-history data from being written.
pkg/backup/backupsink/sst_sink_key_writer.go:43
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **File-size setting becomes stale** This snapshots the public backup file-size setting for the writer's entire lifetime, while the existing `FileSSTSink` reads the setting during writes. If an operator changes `bulkio.backup.file_size` while a reusable key writer is active, it continues using the old threshold and produces files that do not follow the current setting.
PR-level
<h3>Greptile Summary</h3> This PR adds an SST sink API that writes individual MVCC keys, manages span metadata, and performs row-safe size-based splitting and flushing. It also factors out span/prefix helpers and expands backup-sink tests. - Adds `SSTSinkKeyWriter` with `Reset`, `WriteKey`, and explicit row-boundary handling. - Reuses `FileSSTSink` for physical SST output and progress reporting. - Makes the manifest span-size threshold test-overridable. - Adds coverage for span extension, ordering, prefix changes, and size-triggered splits. <h3>Confidence Score: 4/5</h3> The PR is not safe to merge until SSTSinkKeyWriter accepts correctly ordered multiple MVCC versions of the same user key. The new writer rejects a valid and common MVCC ordering case because it discards timestamps during its ordering check; it also snapshots a runtime setting instead of honoring later updates. **Files Needing Attention:** pkg/backup/backupsink/sst_sink_key_writer.go <details><summary><h3>Important Files Changed</h3></summary> | Filename | Overview | |----------|----------| | pkg/backup/backupsink/sst_sink_key_writer.go | Adds the key-at-a-time SST writer; its ordering check rejects valid multi-version MVCC keys and its file-size setting can become stale. | | pkg/backup/backupsink/sst_sink_key_writer_test.go | Adds broad span and flush coverage but does not exercise repeated MVCC versions of one user key or runtime setting changes. | | pkg/backup/backupsink/file_sst_sink.go | Exposes Flush directly, renames elided-prefix state, and makes the manifest span-size limit test-overridable. | | pkg/backup/backupsink/sink_utils.go | Adds focused helpers for span contiguity and elided-prefix equality. | </details> <details open><summary><h3>Flowchart</h3></summary> ```mermaid %%{init: {'theme': 'neutral'}}%% flowchart TD A[Reset with backup span] --> B{Existing SST can be reused?} B -- No --> C[Flush and open SST] B -- Yes --> D[Extend or append manifest span] C --> D D --> E[WriteKey] E --> F[Validate span and ordering] F --> G[Determine previous row boundary] G --> H{Size threshold reached outside row?} H -- Yes --> I[Split manifest span or flush SST] H -- No --> J[Elide prefix and write MVCC entry] I --> J J --> K[Update counts and previous key] K --> L[AssumeNotMidRow or next WriteKey] ``` </details> <!-- greptile_other_comments_section --> <sub>Reviews (1): Last reviewed commit: ["Update pkg/backup/backupsink/BUILD.bazel..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/cab9b6bb9ca4a2ec8b9bae02442f1a2ccc3e363c) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=64284303)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✓ exact
d3 medium ✓ off by one
valid · matches d3 · impact medium
valid · impact high
`StripTablePrefix` removes only the tenant and table ID, so an exclusive EndKey like /Table/6 has the elided prefix /Table/6. A previous span [/Table/5/1, /Table/6) followed by [/Table/6, /Table/7) therefore passes the prefix check, and because the spans are contiguous the file span is extended across the table boundary. `WriteKey` then strips each key's own prefix, so keys from table 5 and table 6 end up in the same elided SST. `FileSSTSink.Write` avoids this by comparing against `s.elidedPrefix`, the prefix of the previous span's start key. This is new code added by this change. [grounded: diff] [consensus 3/3: valid=3]
valid · matches d2 · impact high
raw comments (3)
pkg/backup/backupsink/sst_sink_key_writer.go:68
[P1] WriteKey rejects multiple MVCC versions of the same key The ordering check compares only `key.Key` (`s.prevKey.Compare(key.Key) ›= 0`), so the timestamp is ignored. When a backup has revision history, the same user key appears several times with descending timestamps. The second version of a key then fails with "key ... must be greater than previous key" even though the MVCC key order is valid. `setMidRowForPrevKey` and the midRow comment already account for revisions of a family, so these versions are clearly meant to be accepted. The check should compare full MVCC keys, keeping the previous timestamp alongside `prevKey`. No test writes two versions of one key, which is why the tests don't catch this.
pkg/backup/backupsink/sst_sink_key_writer.go:180
[P1] Prefix check uses the last span's EndKey, so spans from different tables/tenants can share one elided SST `maybeFlushNonExtendableSST` compares the elided prefix of `newSpan.Key` against `lastFile.Span.EndKey`. `FileSSTSink.Write` instead compares against the prefix of the previous span's start key (`s.elidedPrefix`). A span's EndKey often belongs to the next table or tenant. For example, with `ElidePrefix_TenantAndTable`, a span `[/Table/5/1, /Table/6)` is followed by `[/Table/6/1, /Table/7)` or `[/Table/6, …)`. The prefixes look equal (`/Table/6`), so no flush happens. If the spans are contiguous, the file span is even extended across the table boundary. `WriteKey` strips each key's own prefix in `elideMVCCKey`, so keys from table 5 and table 6 end up in the same SST with their table IDs removed. That can break SST ordering or produce a file that restore can't un-elide correctly. The comparison should use the elided prefix of the last span's start key, i.e. `s.elidedPrefix`.
pkg/backup/backupsink/sst_sink_key_writer.go:178
[P1] Prefix check uses the last span's end key rather than the file's elided prefix `maybeFlushNonExtendableSST` checks the new span's prefix against the prefix of `lastFile.Span.EndKey`. It should check it against the prefix the file was opened with (`s.elidedPrefix`, which is the prefix of `lastFile.Span.Key`). The old `FileSSTSink.Write` compared against `s.elidedPrefix`. This breaks when the last span ends exactly at the start of the next prefix. For example, with TenantAndTable mode, take `/Table/1`–`/Table/2` followed by `/Table/2`–`/Table/3`. The two prefixes look equal and the spans are contiguous, so there is no flush, and `Reset` extends the file to `/Table/1`–`/Table/3` while `s.elidedPrefix` stays `/Table/1`. `elideMVCCKey` then strips each key's own prefix (`/Table/2`). Restore rebuilds keys from the prefix of `entry.Span.Key` (`restore_data_processor.go:461`), so it would add `/Table/1` back onto the table 2 keys and restore corrupted data. `elideMVCCKey` works out the prefix from each key instead of checking it against `s.elidedPrefix` the way `copyPointKeys` does. Because of that, this mismatch is never caught by an assertion.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✓ off by one
d3 medium ✓ exact
valid · impact medium
StripTablePrefix strips only tenant+table ID, so a last span ending at /Table/111 has the same prefix as a new span starting at /Table/111. That means no flush happens. Reset then takes the contiguous-extend path without updating elidedPrefix, and WriteKey elides each key with its own prefix, so keys from two tables end up mixed in one SST. FileSSTSink.Write compares against s.elidedPrefix instead and would flush here. [grounded: tree pkg/backup/backupsink/sst_sink_key_writer.go] [consensus 3/3: valid=3]
valid · impact low
WriteKey always sets midRow=true after a write, and the EnsureSafeSplitKey error branch returns without clearing it. This disagrees with adjustFileEndKey, which returns not-mid-row in the same case. As a result maybeDoSizeFlush keeps returning early for such keys. The finding's example of non-table keys is wrong (GetRowPrefixLength returns no error for them), but the error branch is still reachable for malformed family suffixes. [grounded: tree pkg/backup/backupsink/sst_sink_key_writer.go] [consensus 3/3: valid=3]
valid · matches d2 · impact high
valid · matches d3 · impact medium
raw comments (4)
pkg/backup/backupsink/sst_sink_key_writer.go:158
[P2] Prefix-continuity check compares against the previous span's EndKey instead of the file's elided prefix `maybeFlushNonExtendableSST` calls `sameElidedPrefix(newSpan.Key, lastFile.Span.EndKey, ...)`. `FileSSTSink.Write` compares the new span's prefix with `s.elidedPrefix`, which is the prefix of the data already in the file. A span's EndKey is often the start of the next table or tenant. Example: the file holds `[/Table/110, /Table/111)` and the next span is `[/Table/111, ...)`. The EndKey prefix and the new key prefix are both 111, so no flush happens. `Reset` then extends the span or appends a new file entry in the same SST. `WriteKey` elides each key with its own prefix through `elideMVCCKey`, so keys from tables 110 and 111 end up in one file. That file has a single manifest elided prefix, so restore would reassemble the keys incorrectly. `ElidedPrefix(EndKey)` can also return an error when the EndKey is a non-table boundary such as a max key. The check should compare against `s.elidedPrefix`, as the existing sink does.
pkg/backup/backupsink/sst_sink_key_writer.go:243
[P2] `setMidRowForPrevKey` leaves midRow=true when the next key does not parse as a safe split key `WriteKey` always ends with `s.midRow = true`. When `keys.EnsureSafeSplitKey(endKey)` returns an error, `setMidRowForPrevKey` returns early without clearing `midRow`. `adjustFileEndKey`, which this code is derived from, returns "not mid-row" in that case, and the code comment says the same. In the new code the previous key stays marked mid-row. `maybeDoSizeFlush` then returns early on every write. Neither the `fileSpanByteLimit` split nor the `targetFileSize` flush can trigger for keys that do not parse as family keys, such as non-table keys or secondary index keys. The SST and file span then grow without bound. The error branch should set `s.midRow = false`.
pkg/backup/backupsink/sst_sink_key_writer.go:179
[P2] Prefix check compares against the last span's EndKey rather than the file's elided prefix `maybeFlushNonExtendableSST` decides whether to keep appending by comparing the elided prefix of `newSpan.Key` with that of `lastFile.Span.EndKey`. The SST's actual elided prefix is `s.elidedPrefix`, which comes from the file's start key. `FileSSTSink.Write` compares against `s.elidedPrefix` for this reason. If the previous span ends exactly at a table boundary, for example `[/Table/59/1, /Table/60)` followed by `[/Table/60, ...)`, the EndKey and the new start key both map to table 60. The sink then reports "same prefix", skips the flush, and extends the file. The new keys are written into an SST whose existing keys had table 59 elided, and the manifest file span now covers two tables. That only happens when a span ends on a table boundary, but a restore would then read the elided keys with the wrong prefix. The check should compare `newSpan.Key`'s prefix with `s.elidedPrefix`.
pkg/backup/backupsink/sst_sink_key_writer.go:69
[P2] Multiple MVCC versions of the same user key are rejected `WriteKey` takes an `MVCCKey` but tracks only `prevKey` as the user key. It returns an assertion error when `prevKey.Compare(key.Key) ›= 0`, so a second revision of the same key with an older timestamp is refused. Backups with revision history emit several consecutive versions of one key. The `midRow` logic in `setMidRowForPrevKey` also describes "revisions of a family", which suggests versions were meant to be supported. A revision-history backup that goes through this writer would fail on the first key with more than one version. The ordering check should compare full MVCC keys, or allow an equal user key with a lower timestamp.
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 critical ✗ —
d2 high ✗ —
d3 medium ✓ off by one
valid · matches d3 · impact medium
valid · impact medium
In storage/sst_writer.go, PutRawMVCC panics with "PutRawMVCC timestamp is empty" when the timestamp is empty. WriteKey has no IsEmpty branch, while copyPointKeys sends empty-timestamp keys to PutUnversioned. So a public WriteKey call with an unversioned key crashes. [grounded: diff] [consensus 3/3: valid=3]
uncertain
WriteKey does not clear the value checksum the way copyPointKeys does. However, the regular restore path (restore_data_processor) calls ClearChecksum and InitChecksum on every key it rewrites, and the expected inputs (values read back from existing backup files) already have their checksums cleared. Whether a verification failure can actually happen is not shown. [uncertain: no evidence quoted] [consensus 2/5: uncertain=2, valid=2, invalid=1]
valid · impact low
The new writer keeps the pacer it is given but never calls Pace in WriteKey, unlike the copyPointKeys loop. Writes through this path skip elastic CPU admission. No caller uses it yet, so the impact is small. [grounded: diff] [consensus 3/3: valid=3]
raw comments (6)
pkg/backup/backupsink/sst_sink_key_writer.go:70
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 1\. Revision-history backups reject valid key versions <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>WriteKey</i></b> retains only the previous <b><i>roachpb.Key</i></b> and calls <b><i>prevKey.Compare(key.Key)</i></b>, discarding the timestamp from the accepted <b><i>storage.MVCCKey</i></b>. Revision-history exports can supply consecutive versions of one logical key in valid descending-timestamp MVCC order, but the second version reaches this assertion instead of the SST writer. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description `WriteKey` rejects correctly ordered MVCC revisions that share a user key because it stores and compares only `roachpb.Key`, ignoring each revision's timestamp. ## Fix Focus Areas - pkg/backup/backupsink/sst_sink_key_writer.go[25-28] - pkg/backup/backupsink/sst_sink_key_writer.go[69-70] - pkg/backup/backupsink/sst_sink_key_writer.go[98-98] - pkg/backup/backupsink/sst_sink_key_writer_test.go[389-409] ## Recommended Fix Store the previous complete `storage.MVCCKey` and compare complete MVCC keys with `storage.MVCCKey.Compare` or `Less` before writing. After each successful write, clone both the key bytes and timestamp into the saved previous key. Add coverage that writes multiple versions of one user key in descending timestamp order, while retaining coverage that genuinely out-of-order MVCC entries are rejected. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
pkg/backup/backupsink/sst_sink_key_writer.go:93
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 2\. Unversioned backup keys crash the writer <code>🐞 Bug</code> <code>☼ Reliability</code> <pre> <b><i>WriteKey</i></b> always calls <b><i>PutRawMVCC</i></b> even when <b><i>elidedKey.Timestamp</i></b> is empty. Unversioned MVCC metadata keys are supported by the storage key type and the existing sink writes them with <b><i>PutUnversioned</i></b>, while <b><i>PutRawMVCC</i></b> panics for that input. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description The new writer sends unversioned MVCC keys to `PutRawMVCC`, which panics instead of writing them. ## Fix Focus Areas - pkg/backup/backupsink/sst_sink_key_writer.go[82-93] - pkg/backup/backupsink/sst_sink_key_writer_test.go[378-439] ## Recommended Fix Branch on `key.Timestamp.IsEmpty()` after prefix elision: call `PutUnversioned(elidedKey.Key, value)` for empty timestamps and retain `PutRawMVCC` for versioned keys. Add a test that writes and flushes an unversioned key without panicking. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
pkg/backup/backupsink/sst_sink_key_writer.go:93
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> 3\. Elided backup files retain wrong checksums <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>WriteKey</i></b> elides the key prefix but passes the supplied raw MVCC value unchanged to <b><i>PutRawMVCC</i></b>, retaining a checksum that covers the original full key. When a restore remaps the backed-up key, verification recomputes the nonzero checksum with the elided or destination key and can reject the restored data. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description `WriteKey` changes the stored key through prefix elision while retaining the raw MVCC value's key-dependent checksum, which can become invalid when the backup is restored under another key. ## Fix Focus Areas - pkg/backup/backupsink/sst_sink_key_writer.go[82-93] - pkg/backup/backupsink/file_sst_sink.go[383-414] ## Recommended Fix Mirror the existing `FileSSTSink.copyPointKeys` path before writing an elided point key: copy and decode the raw MVCC value, clear its checksum while preserving any MVCC value header, and pass the resulting raw bytes to `PutRawMVCC`. Add a restore-oriented test whose input value is checksummed with the original full key and verify it after key remapping. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
pkg/backup/backupsink/sst_sink_key_writer.go:92
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> 4\. Backup writes bypass admission pacing <code>🐞 Bug</code> <code>➹ Performance</code> <pre> <b><i>MakeSSTSinkKeyWriter</i></b> stores the supplied admission pacer through its embedded <b><i>FileSSTSink</i></b>, but <b><i>WriteKey</i></b> performs each per-key SST write without calling <b><i>s.pacer.Pace(ctx)</i></b>. When backup or compaction output repeatedly invokes this path with elastic processor admission active, its tight write loop bypasses the cooperative scheduling used by the existing backup key-copy loop. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description The key writer accepts and stores an admission pacer but never invokes it while processing keys, allowing its tight SST write path to bypass elastic work throttling. ## Fix Focus Areas - pkg/backup/backupsink/sst_sink_key_writer.go[56-105] - pkg/backup/backupsink/file_sst_sink.go[363-369] ## Recommended Fix Call `s.pacer.Pace(ctx)` once for every `WriteKey` invocation before performing the SST write work, and propagate pacing cancellation or admission errors to the caller as the existing point-key loop does. Add a test using a controllable pacer if the package provides suitable admission test utilities. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
PR-level
<h3>PR Summary by Qodo</h3> Add key-at-a-time backup SST sink writer <code>✨ Enhancement</code> <code>🧪 Tests</code> <code>🕐 40+ Minutes</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <details> <summary>AI Description</summary> <dl> <dd> <br/> ><pre> >• Add a key-oriented SST writer reusing backup flushing and progress reporting. >• Enforce ordered writes, prefix elision, row-safe splitting, and size limits. >• Cover span extension, flush boundaries, invalid writes, and mid-row safeguards. ></pre> </dd> </dl> </details> <details> <summary>Diagram</summary> <dl> <dd> <br/> ```mermaid graph TD A["Backup Caller"] -->|Reset span| B["Key Writer"] -->|Validate key| C["Prefix Elision"] -->|Write MVCC| D["SST Writer"] -->|Flush file| E[("External Storage")] B -->|Emit metadata| F["Backup Progress"] ``` </dd> </dl> </details> <details> <summary>High-Level Assessment</summary> <dl> <dd> <br/> >The following are alternative approaches to this PR: <details> <summary><b>1. Extract a shared sink core</b></summary> <dl> <dd> - ➕ Avoids embedding and directly manipulating FileSSTSink internals - ➕ Creates clearer ownership boundaries for flushing, metadata, and SST lifecycle - ➕ Makes future sink variants easier to add - ➖ Requires a broader refactor of the established FileSSTSink path - ➖ Expands regression risk beyond the new key-writing use case - ➖ Adds review scope before the abstraction has multiple proven consumers </dd> </dl> </details> >**Recommendation:** Keep the PR&#x27;s incremental approach: embedding FileSSTSink efficiently reuses its persistence, progress, encryption, and lifecycle behavior while limiting changes to the existing export path. Consider extracting a shared internal sink core later if additional writer modes need the same internals. </dd> </dl> </details> <details> <summary> Files changed (6) <code> +921 / -45 </code> </summary> <dl> <dd> <br/> <details> <summary>Enhancement (2) <code> +296 / -0 </code></summary> <dl> <dd> <details> <summary>sink_utils.go<code>Add span and elided-prefix comparison helpers</code> <code>+20/-0</code></summary> <br/> >Add span and elided-prefix comparison helpers > ><pre> >• Adds helpers for detecting contiguous spans and comparing key prefixes under the configured elision mode. The key writer uses them to determine whether metadata can be extended or the current SST must flush. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-fd75a2f66e905300d50539617d7736ff364b93662a3171a9cde0316e525c5f30'>pkg/backup/backupsink/sink_utils.go</a> </details> <details> <summary>sst_sink_key_writer.go<code>Implement ordered per-key SST writing</code> <code>+276/-0</code></summary> <br/> >Implement ordered per-key SST writing > ><pre> >• Introduces SSTSinkKeyWriter for writing individual ordered MVCC keys into backup SSTs with mandatory span resets. It manages prefix elision, manifest span extension and splitting, target-size flushing, out-of-order protection, and explicit mid-row safety. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-18f20814af84c3b873a869814385021526102e8997c32257f2d3b29975b8477c'>pkg/backup/backupsink/sst_sink_key_writer.go</a> </details> </dd> </dl> </details> <details> <summary>Refactor (1) <code> +22 / -24 </code></summary> <dl> <dd> <details> <summary>file_sst_sink.go<code>Expose reusable flush behavior and sink state</code> <code>+22/-24</code></summary> <br/> >Expose reusable flush behavior and sink state > ><pre> >• Simplifies Flush into the primary implementation, renames the tracked elided prefix, and makes the manifest span byte limit test-overridable. These adjustments allow SSTSinkKeyWriter to reuse FileSSTSink lifecycle and persistence behavior. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-5b818e1f05e53cbcc8b508590419ea9fa96999536eb5abb5b2bb4a721124485a'>pkg/backup/backupsink/file_sst_sink.go</a> </details> </dd> </dl> </details> <details> <summary>Tests (2) <code> +597 / -20 </code></summary> <dl> <dd> <details> <summary>file_sst_sink_test.go<code>Share configurable sink test setup</code> <code>+39/-20</code></summary> <br/> >Share configurable sink test setup > ><pre> >• Refactors test setup to accept prefix-elision modes and return reusable sink configuration. Adds MVCC test helpers needed by the key writer suite and updates existing callers. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-ecc715916cac85b4750d0c99baf93e5428e8339509948c8a17ea6a92790b34ae'>pkg/backup/backupsink/file_sst_sink_test.go</a> </details> <details> <summary>sst_sink_key_writer_test.go<code>Validate key writer span and flush invariants</code> <code>+558/-0</code></summary> <br/> >Validate key writer span and flush invariants > ><pre> >• Adds comprehensive coverage for size-based flushes, span extension and ordering, prefix changes, manifest limits, and mid-row handling. It also verifies failures for unordered keys, out-of-span keys, missing resets, and unsafe flushes. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-53a3870a60a42ab874803325e48b7b6ef53ee9a73c64426a2dbca0657e960750'>pkg/backup/backupsink/sst_sink_key_writer_test.go</a> </details> </dd> </dl> </details> <details> <summary>Other (1) <code> +6 / -1 </code></summary> <dl> <dd> <details> <summary>BUILD.bazel<code>Register key writer sources and tests</code> <code>+6/-1</code></summary> <br/> >Register key writer sources and tests > ><pre> >• Adds the new key writer implementation and test file to the backupsink Bazel targets. Adds the testutils dependency used to override the file span limit during tests. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-14216e9567ae7efb04457cda9fd958253f677eb2e06da7ea93eb0a33cb3a7926'>pkg/backup/backupsink/BUILD.bazel</a> </details> </dd> </dl> </details> </dd> </dl> </details>
PR-level
<h3>Code Review by Qodo</h3> <code>🐞 Bugs (4)</code> <code>📘 Rule violations (0)</code> <code>📜 Skill insights (0)</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <br/> <img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> <details> <summary> 1. Revision-history backups reject valid key versions <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>WriteKey</i></b> retains only the previous <b><i>roachpb.Key</i></b> and calls <b><i>prevKey.Compare(key.Key)</i></b>, discarding >the timestamp from the accepted <b><i>storage.MVCCKey</i></b>. Revision-history exports can supply consecutive >versions of one logical key in valid descending-timestamp MVCC order, but the second version reaches >this assertion instead of the SST writer. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/backup/backupsink/sst_sink_key_writer.go[R69-70]](https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-18f20814af84c3b873a869814385021526102e8997c32257f2d3b29975b8477cR69-R70)</code> > >```diff >+ if s.prevKey != nil && s.prevKey.Compare(key.Key) >= 0 { >+ return errors.AssertionFailedf("key %s must be greater than previous key %s", key.Key, s.prevKey) >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The writer accepts a complete <b><i>storage.MVCCKey</i></b>, but the added state and comparison retain only its >user-key bytes. The repository defines MVCC ordering by user key followed by reverse timestamp >order, and revision-export code explicitly groups multiple timestamped values under the same user >key, proving that equal user keys with descending timestamps are valid input. ></pre> > > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[56-70]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L56-L70)</code> > <code>[pkg/storage/mvcc_key.go[83-97]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/storage/mvcc_key.go/#L83-L97)</code> > <code>[pkg/backup/revision_reader.go[18-24]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/revision_reader.go/#L18-L24)</code> > <code>[pkg/backup/revision_reader.go[78-92]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/revision_reader.go/#L78-L92)</code> > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[25-28]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L25-L28)</code> > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[92-98]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L92-L98)</code> > <code>[pkg/storage/mvcc_key.go[37-48]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/storage/mvcc_key.go/#L37-L48)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >`WriteKey` rejects correctly ordered MVCC revisions that share a user key because it stores and compares only `roachpb.Key`, ignoring each revision's timestamp. > >## Fix Focus Areas >- pkg/backup/backupsink/sst_sink_key_writer.go[25-28] >- pkg/backup/backupsink/sst_sink_key_writer.go[69-70] >- pkg/backup/backupsink/sst_sink_key_writer.go[98-98] >- pkg/backup/backupsink/sst_sink_key_writer_test.go[389-409] > >## Recommended Fix >Store the previous complete `storage.MVCCKey` and compare complete MVCC keys with `storage.MVCCKey.Compare` or `Less` before writing. After each successful write, clone both the key bytes and timestamp into the saved previous key. Add coverage that writes multiple versions of one user key in descending timestamp order, while retaining coverage that genuinely out-of-order MVCC entries are rejected. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <details> <summary> 2. Unversioned backup keys crash the writer <code>🐞 Bug</code> <code>☼ Reliability</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>WriteKey</i></b> always calls <b><i>PutRawMVCC</i></b> even when <b><i>elidedKey.Timestamp</i></b> is empty. Unversioned MVCC >metadata keys are supported by the storage key type and the existing sink writes them with ><b><i>PutUnversioned</i></b>, while <b><i>PutRawMVCC</i></b> panics for that input. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/backup/backupsink/sst_sink_key_writer.go[R82-93]](https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-18f20814af84c3b873a869814385021526102e8997c32257f2d3b29975b8477cR82-R93)</code> > >```diff >+ elidedKey, err := elideMVCCKey(key, s.conf.ElideMode) >+ if err != nil { >+ return err >+ } >+ // TODO (kev-cao): Should add some method of tracking `Rows` (and potentially >+ // `IndexEntries`). >+ keyAsRowCount := roachpb.RowCount{ >+ DataSize: int64(len(key.Key)) + int64(len(value)), >+ } >+ >+ if err := s.sst.PutRawMVCC(elidedKey, value); err != nil { >+ return err >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The storage writer explicitly panics when <b><i>PutRawMVCC</i></b> receives an empty timestamp. The existing >FileSSTSink handles precisely this case by selecting <b><i>PutUnversioned</i></b>, demonstrating that this is a >supported point-key form for backup SST copying. ></pre> > > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[82-93]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L82-L93)</code> > <code>[pkg/storage/mvcc_key.go[46-54]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/storage/mvcc_key.go/#L46-L54)</code> > <code>[pkg/storage/sst_writer.go[403-419]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/storage/sst_writer.go/#L403-L419)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[394-414]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L394-L414)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >The new writer sends unversioned MVCC keys to `PutRawMVCC`, which panics instead of writing them. > >## Fix Focus Areas >- pkg/backup/backupsink/sst_sink_key_writer.go[82-93] >- pkg/backup/backupsink/sst_sink_key_writer_test.go[378-439] > >## Recommended Fix >Branch on `key.Timestamp.IsEmpty()` after prefix elision: call `PutUnversioned(elidedKey.Key, value)` for empty timestamps and retain `PutRawMVCC` for versioned keys. Add a test that writes and flushes an unversioned key without panicking. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <br/> <img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> <details> <summary> 3. Elided backup files retain wrong checksums <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>WriteKey</i></b> elides the key prefix but passes the supplied raw MVCC value unchanged to <b><i>PutRawMVCC</i></b>, >retaining a checksum that covers the original full key. When a restore remaps the backed-up key, >verification recomputes the nonzero checksum with the elided or destination key and can reject the >restored data. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/backup/backupsink/sst_sink_key_writer.go[R82-93]](https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-18f20814af84c3b873a869814385021526102e8997c32257f2d3b29975b8477cR82-R93)</code> > >```diff >+ elidedKey, err := elideMVCCKey(key, s.conf.ElideMode) >+ if err != nil { >+ return err >+ } >+ // TODO (kev-cao): Should add some method of tracking `Rows` (and potentially >+ // `IndexEntries`). >+ keyAsRowCount := roachpb.RowCount{ >+ DataSize: int64(len(key.Key)) + int64(len(value)), >+ } >+ >+ if err := s.sst.PutRawMVCC(elidedKey, value); err != nil { >+ return err >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The new path changes the key while writing the value unchanged, whereas the established backup sink >explicitly clears checksums before putting elided keys because exported values may no longer reside >at their original key. <b><i>roachpb.Value.InitChecksum</i></b> shows that the checksum includes the supplied >key, and <b><i>roachpb.Value.Verify</i></b> recomputes any nonzero checksum using the destination key and >returns an error when it differs. ></pre> > > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[82-93]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L82-L93)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[383-414]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L383-L414)</code> > <code>[pkg/roachpb/data.go[330-349]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/roachpb/data.go/#L330-L349)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[383-397]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L383-L397)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[399-412]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L399-L412)</code> > <code>[pkg/roachpb/data.go[330-360]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/roachpb/data.go/#L330-L360)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >`WriteKey` changes the stored key through prefix elision while retaining the raw MVCC value's key-dependent checksum, which can become invalid when the backup is restored under another key. > >## Fix Focus Areas >- pkg/backup/backupsink/sst_sink_key_writer.go[82-93] >- pkg/backup/backupsink/file_sst_sink.go[383-414] > >## Recommended Fix >Mirror the existing `FileSSTSink.copyPointKeys` path before writing an elided point key: copy and decode the raw MVCC value, clear its checksum while preserving any MVCC value header, and pass the resulting raw bytes to `PutRawMVCC`. Add a restore-oriented test whose input value is checksummed with the original full key and verify it after key remapping. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <details> <summary> 4. Backup writes bypass admission pacing <code>🐞 Bug</code> <code>➹ Performance</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>MakeSSTSinkKeyWriter</i></b> stores the supplied admission pacer through its embedded <b><i>FileSSTSink</i></b>, but ><b><i>WriteKey</i></b> performs each per-key SST write without calling <b><i>s.pacer.Pace(ctx)</i></b>. When backup or >compaction output repeatedly invokes this path with elastic processor admission active, its tight >write loop bypasses the cooperative scheduling used by the existing backup key-copy loop. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/backup/backupsink/sst_sink_key_writer.go[R78-92]](https://github.com/drape-io/crbench-qodo-2026-09/pull/26/files#diff-18f20814af84c3b873a869814385021526102e8997c32257f2d3b29975b8477cR78-R92)</code> > >```diff >+ if err := s.maybeDoSizeFlush(ctx, key.Key); err != nil { >+ return err >+ } >+ >+ elidedKey, err := elideMVCCKey(key, s.conf.ElideMode) >+ if err != nil { >+ return err >+ } >+ // TODO (kev-cao): Should add some method of tracking `Rows` (and potentially >+ // `IndexEntries`). >+ keyAsRowCount := roachpb.RowCount{ >+ DataSize: int64(len(key.Key)) + int64(len(value)), >+ } >+ >+ if err := s.sst.PutRawMVCC(elidedKey, value); err != nil { >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >The constructor creates and embeds a <b><i>FileSSTSink</i></b> with the supplied pacer, while the complete ><b><i>WriteKey</i></b> path never invokes it. The pacer contract requires callers in tight loops to call <b><i>Pace</i></b> >on every iteration, and the established backup point-key SST iteration demonstrates that expected >usage. ></pre> > > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[35-44]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L35-L44)</code> > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[78-103]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L78-L103)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[365-369]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L365-L369)</code> > <code>[pkg/util/admission/pacer.go[13-17]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/util/admission/pacer.go/#L13-L17)</code> > <code>[pkg/util/admission/pacer.go[26-45]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/util/admission/pacer.go/#L26-L45)</code> > <code>[pkg/backup/backupsink/sst_sink_key_writer.go[56-105]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/sst_sink_key_writer.go/#L56-L105)</code> > <code>[pkg/backup/backupsink/file_sst_sink.go[363-369]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/backup/backupsink/file_sst_sink.go/#L363-L369)</code> > <code>[pkg/util/admission/pacer.go[26-44]](https://github.com/drape-io/crbench-qodo-2026-09/blob/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39/pkg/util/admission/pacer.go/#L26-L44)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >The key writer accepts and stores an admission pacer but never invokes it while processing keys, allowing its tight SST write path to bypass elastic work throttling. > >## Fix Focus Areas >- pkg/backup/backupsink/sst_sink_key_writer.go[56-105] >- pkg/backup/backupsink/file_sst_sink.go[363-369] > >## Recommended Fix >Call `s.pacer.Pace(ctx)` once for every `WriteKey` invocation before performing the SST write work, and propagate pacing cancellation or admission errors to the caller as the existing point-key loop does. Add a test using a controllable pacer if the package provides suitable admission test utilities. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-context:start --> <details><summary><strong>Context sources</strong></summary> <div>Review mode: <code>🧠 Deep</code>: This introduces substantial new backup/SST-writing logic across multiple interacting paths, with numerous independent edge cases around ordering, prefix elision, span metadata, row boundaries, and flushing that merit redundant review.</div> <!-- qodo-context:end --> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:start --> <details> <summary><strong>Tip of the day</strong></summary> <br/> <pre>💡 Did you know, you can reply &#x27;qodo&#x27; on any finding to push back, ask questions, or dig deeper</pre> <a href="https://docs.qodo.ai/tips-and-tricks">More tips ↗</a> | <a href="https://app.qodo.ai/configurations?tab=display-preferences">Customize Qodo ↗</a> | <a href="https://docs.qodo.ai">Qodo docs ↗</a> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:end --> <!-- https://github.com/drape-io/crbench-qodo-2026-09/commit/3b5db7b1d4ee0d99bae870b9ecf87ef8611fed39 --> <a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>