crbench//

← all cases

Unified Storage Performance Optimizations

Tier 2 medium go · local · real merged PR from grafana/grafana #97529

Ground-truth defects

d1 · other · high
A race condition in BuildIndex allows multiple goroutines to concurrently build the same expensive index for the same key. This is caused by moving the b.cacheMu lock from protecting the entire function to only protecting the final cache assignment.
d2 · other · medium
Calling s.search.TotalDocs() here may race with concurrent index creation: TotalDocs iterates b.cache without synchronization, and the event watcher goroutine started just above could trigger BuildIndex writes concurrently, potentially causing a concurrent map read/write panic.

The change under review

diff
diff --git a/pkg/server/module_server_test.go b/pkg/server/module_server_test.go
index 272fe56c53..ec7b97b123 100644
--- a/pkg/server/module_server_test.go
+++ b/pkg/server/module_server_test.go
@@ -32,6 +32,10 @@ func TestIntegrationWillRunInstrumentationServerWhenTargetHasNoHttpServer(t *tes
 	if dbType == "sqlite3" {
 		t.Skip("skipping - sqlite not supported for storage server target")
 	}
+	// TODO - fix this test for postgres
+	if dbType == "postgres" {
+		t.Skip("skipping - test not working with postgres in Drone. Works locally.")
+	}
 
 	_, cfg := db.InitTestDBWithCfg(t)
 	cfg.HTTPPort = "3001"
diff --git a/pkg/storage/unified/resource/search.go b/pkg/storage/unified/resource/search.go
index 9ea3a321a3..341470812a 100644
--- a/pkg/storage/unified/resource/search.go
+++ b/pkg/storage/unified/resource/search.go
@@ -8,7 +8,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -171,7 +170,7 @@ func (s *searchSupport) Search(ctx context.Context, req *ResourceSearchRequest)
 
 // init is called during startup.  any failure will block startup and continued execution
 func (s *searchSupport) init(ctx context.Context) error {
-	_, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init")
+	ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Init")
 	defer span.End()
 	start := time.Now().Unix()
 
@@ -214,6 +213,7 @@ func (s *searchSupport) init(ctx context.Context) error {
 	}()
 
 	end := time.Now().Unix()
+	s.log.Info("search index initialized", "duration_secs", end-start, "total_docs", s.search.TotalDocs())
 	if IndexMetrics != nil {
 		IndexMetrics.IndexCreationTime.WithLabelValues().Observe(float64(end - start))
 	}
@@ -277,7 +277,7 @@ func (s *searchSupport) handleEvent(ctx context.Context, evt *WrittenEvent) {
 	// record latency from when event was created to when it was indexed
 	latencySeconds := float64(time.Now().UnixMicro()-evt.ResourceVersion) / 1e6
 	if latencySeconds > 5 {
-		logger.Warn("high index latency", "latency", latencySeconds)
+		s.log.Warn("high index latency", "latency", latencySeconds)
 	}
 	if IndexMetrics != nil {
 		IndexMetrics.IndexLatency.WithLabelValues(evt.Key.Resource).Observe(latencySeconds)
@@ -307,7 +307,7 @@ func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedReso
 }
 
 func (s *searchSupport) build(ctx context.Context, nsr NamespacedResource, size int64, rv int64) (ResourceIndex, int64, error) {
-	_, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Build")
+	ctx, span := s.tracer.Start(ctx, tracingPrexfixSearch+"Build")
 	defer span.End()
 
 	builder, err := s.builders.get(ctx, nsr)
diff --git a/pkg/storage/unified/resource/server.go b/pkg/storage/unified/resource/server.go
index 8339deefbc..8d17113566 100644
--- a/pkg/storage/unified/resource/server.go
+++ b/pkg/storage/unified/resource/server.go
@@ -255,6 +255,12 @@ func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) {
 		}
 	}
 
+	err := s.Init(ctx)
+	if err != nil {
+		s.log.Error("error initializing resource server", "error", err)
+		return nil, err
+	}
+
 	return s, nil
 }
 
@@ -294,16 +300,16 @@ func (s *server) Init(ctx context.Context) error {
 			}
 		}
 
-		// Start watching for changes
-		if s.initErr == nil {
-			s.initErr = s.initWatcher()
-		}
-
 		// initialize the search index
 		if s.initErr == nil && s.search != nil {
 			s.initErr = s.search.init(ctx)
 		}
 
+		// Start watching for changes
+		if s.initErr == nil {
+			s.initErr = s.initWatcher()
+		}
+
 		if s.initErr != nil {
 			s.log.Error("error initializing resource server", "error", s.initErr)
 		}
@@ -446,10 +452,6 @@ func (s *server) Create(ctx context.Context, req *CreateRequest) (*CreateRespons
 	ctx, span := s.tracer.Start(ctx, "storage_server.Create")
 	defer span.End()
 
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
-
 	rsp := &CreateResponse{}
 	user, ok := claims.From(ctx)
 	if !ok || user == nil {
@@ -488,10 +490,6 @@ func (s *server) Update(ctx context.Context, req *UpdateRequest) (*UpdateRespons
 	ctx, span := s.tracer.Start(ctx, "storage_server.Update")
 	defer span.End()
 
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
-
 	rsp := &UpdateResponse{}
 	user, ok := claims.From(ctx)
 	if !ok || user == nil {
@@ -542,10 +540,6 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
 	ctx, span := s.tracer.Start(ctx, "storage_server.Delete")
 	defer span.End()
 
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
-
 	rsp := &DeleteResponse{}
 	if req.ResourceVersion < 0 {
 		return nil, apierrors.NewBadRequest("update must include the previous version")
@@ -634,9 +628,6 @@ func (s *server) Delete(ctx context.Context, req *DeleteRequest) (*DeleteRespons
 }
 
 func (s *server) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, error) {
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	user, ok := claims.From(ctx)
 	if !ok || user == nil {
 		return &ReadResponse{
@@ -693,9 +684,6 @@ func (s *server) List(ctx context.Context, req *ListRequest) (*ListResponse, err
 			}}, nil
 	}
 
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	if req.Limit < 1 {
 		req.Limit = 50 // default max 50 items in a page
 	}
@@ -786,10 +774,6 @@ func (s *server) initWatcher() error {
 func (s *server) Watch(req *WatchRequest, srv ResourceStore_WatchServer) error {
 	ctx := srv.Context()
 
-	if err := s.Init(ctx); err != nil {
-		return err
-	}
-
 	user, ok := claims.From(ctx)
 	if !ok || user == nil {
 		return apierrors.NewUnauthorized("no user found in context")
@@ -930,9 +914,6 @@ func (s *server) Watch(req *WatchRequest, srv ResourceStore_WatchServer) error {
 }
 
 func (s *server) Search(ctx context.Context, req *ResourceSearchRequest) (*ResourceSearchResponse, error) {
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	if s.search == nil {
 		return nil, fmt.Errorf("search index not configured")
 	}
@@ -941,25 +922,16 @@ func (s *server) Search(ctx context.Context, req *ResourceSearchRequest) (*Resou
 
 // History implements ResourceServer.
 func (s *server) History(ctx context.Context, req *HistoryRequest) (*HistoryResponse, error) {
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	return s.search.History(ctx, req)
 }
 
 // Origin implements ResourceServer.
 func (s *server) Origin(ctx context.Context, req *OriginRequest) (*OriginResponse, error) {
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	return s.search.Origin(ctx, req)
 }
 
 // IsHealthy implements ResourceServer.
 func (s *server) IsHealthy(ctx context.Context, req *HealthCheckRequest) (*HealthCheckResponse, error) {
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 	return s.diagnostics.IsHealthy(ctx, req)
 }
 
@@ -971,9 +943,6 @@ func (s *server) PutBlob(ctx context.Context, req *PutBlobRequest) (*PutBlobResp
 			Code:    http.StatusNotImplemented,
 		}}, nil
 	}
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
 
 	rsp, err := s.blob.PutResourceBlob(ctx, req)
 	if err != nil {
@@ -1016,10 +985,6 @@ func (s *server) GetBlob(ctx context.Context, req *GetBlobRequest) (*GetBlobResp
 		}}, nil
 	}
 
-	if err := s.Init(ctx); err != nil {
-		return nil, err
-	}
-
 	// The linked blob is stored in the resource metadata attributes
 	obj, status := s.getPartialObject(ctx, req.Resource, req.ResourceVersion)
 	if status != nil {
diff --git a/pkg/storage/unified/search/bleve.go b/pkg/storage/unified/search/bleve.go
index b8047142fe..932c69e312 100644
--- a/pkg/storage/unified/search/bleve.go
+++ b/pkg/storage/unified/search/bleve.go
@@ -85,9 +85,6 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 	// The builder will write all documents before returning
 	builder func(index resource.ResourceIndex) (int64, error),
 ) (resource.ResourceIndex, error) {
-	b.cacheMu.Lock()
-	defer b.cacheMu.Unlock()
-
 	_, span := b.tracer.Start(ctx, tracingPrexfixBleve+"BuildIndex")
 	defer span.End()
 
@@ -99,9 +96,9 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 	if size > b.opts.FileThreshold {
 		dir := filepath.Join(b.opts.Root, key.Namespace, fmt.Sprintf("%s.%s", key.Resource, key.Group))
 		index, err = bleve.New(dir, mapper)
-		if err == nil {
-			b.log.Info("TODO, check last RV so we can see if the numbers have changed", "dir", dir)
-		}
+
+		// TODO, check last RV so we can see if the numbers have changed
+
 		resource.IndexMetrics.IndexTenants.WithLabelValues(key.Namespace, "file").Inc()
 	} else {
 		index, err = bleve.NewMemOnly(mapper)
@@ -137,7 +134,9 @@ func (b *bleveBackend) BuildIndex(ctx context.Context,
 		return nil, err
 	}
 
+	b.cacheMu.Lock()
 	b.cache[key] = idx
+	b.cacheMu.Unlock()
 	return idx, nil
 }
 
diff --git a/pkg/storage/unified/sql/backend.go b/pkg/storage/unified/sql/backend.go
index ae20787016..965930dcf5 100644
--- a/pkg/storage/unified/sql/backend.go
+++ b/pkg/storage/unified/sql/backend.go
@@ -123,7 +123,7 @@ func (b *backend) Stop(_ context.Context) error {
 
 // GetResourceStats implements Backend.
 func (b *backend) GetResourceStats(ctx context.Context, namespace string, minCount int) ([]resource.ResourceStats, error) {
-	_, span := b.tracer.Start(ctx, tracePrefix+".GetResourceStats")
+	ctx, span := b.tracer.Start(ctx, tracePrefix+".GetResourceStats")
 	defer span.End()
 
 	req := &sqlStatsRequest{

Leaderboard for this case

medium · 5 of 9 bots fully caught it (56% solve rate) · 72% avg coverage.

#BotResultCaughtCoverageFindingsFalse alarms
1 Claude Code (Opus 5.5) solved 2/2 100.0% 4 0
2 Cubic solved 2/2 100.0% 3 0
3 cursor solved 2/2 100.0% 2 0
4 Drape (Deep Review) solved 2/2 100.0% 3 0
5 Drape (Review) solved 2/2 100.0% 3 0
6 qodo partial 1/2 66.7% 2 0
7 Mjolnir (Opus 5.5) partial 1/2 33.3% 3 0
8 Mjolnir (Sonnet 5.5) partial 1/2 33.3% 1 0
9 greptile miss 0/2 0.0% 2 0

Bot results

Claude Code (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ off by one
d2 medium ✓ exact
valid · matches d2 · impact medium
valid · matches d1 · impact high
valid · impact medium
Before the change, every handler called s.Init, which returns s.initErr. Stop() sets initErr to 'service is stopping' or 'service is stopped', so requests made after Stop were rejected. This change removes those per-request calls, so requests now run against a stopped backend. Init also now runs synchronously in the constructor, and any failure there fails construction. [grounded: diff] [consensus 3/3: valid=3]
valid · impact low
Init already logs 'error initializing resource server' with s.initErr. The new constructor code logs the same message again for the same error, so each failure is logged twice. The effect is minor log noise. [grounded: diff] [consensus 3/3: valid=3]
raw comments (4)
pkg/storage/unified/resource/search.go:216
The new log line calls s.search.TotalDocs(). That happens after the event-watcher goroutine has started, and TotalDocs in bleve.go reads b.cache without holding cacheMu. Meanwhile handleEvent -> getOrCreateIndex -> BuildIndex can write to b.cache from that goroutine, and so can concurrent Search requests. The result is an unsynchronized map read racing a map write, which Go can abort as a fatal 'concurrent map read and map write'. TotalDocs needs to take cacheMu.RLock.
pkg/storage/unified/search/bleve.go:88
cacheMu is no longer held for the whole BuildIndex, so builds for the same key can now run at the same time. Search, handleEvent and init can all see a nil index and build in parallel. For file-backed indexes, both builds call bleve.New on the same directory, so one fails with 'path already exists' or they corrupt each other's on-disk index. In every case the last writer silently replaces the cached index, so watch events written to the losing index are lost and IndexTenants is counted twice. This needs per-key locking or a check-then-build done under the lock.
pkg/storage/unified/resource/server.go:258
Init now runs inside the constructor and is no longer checked per request. As a result, Stop() setting s.initErr ('service is stopping'/'service is stopped') no longer stops anything: Create/Update/Delete/Read/List/Watch/Search/blob calls keep running against a backend whose lifecycle has been stopped and whose watch context has been cancelled. Startup is also slower and less robust. Constructing the server now synchronously runs lifecycle Init (DB migrations) and builds every search index. Any failure there makes NewResourceServer fail outright instead of surfacing on requests, and callers such as the gRPC service create it before their own start phase.
pkg/storage/unified/resource/server.go:260
When initialization fails, the error is logged twice: Init already logs 'error initializing resource server', and NewResourceServer logs the same message again.
Cubic · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ off by one
d2 medium ✓ exact
valid · matches d1 · impact high
valid · impact low
The new eager `s.Init(ctx)` call in NewResourceServer returns nil on error. It does not call `cancel()` or `lifecycle.Stop`, and search's background watch goroutine (started with context.Background) keeps running. Because the server is returned as nil, the caller can't call Stop to clean up either. Before this change, Init ran lazily and the server object stayed available for Stop. [grounded: diff] [consensus 5/5: valid=5]
valid · matches d2 · impact medium
raw comments (4)
pkg/storage/unified/search/bleve.go:137
<!-- cubic:v=c59e519f-9fce-467e-8605-ea0f60122869 --> <!-- cubic:review-run=c1590f03-341c-591a-b20a-dd2f76bbb7b9 --> <!-- metadata:{"confidence":9} --> P2: When concurrent requests initialize the same key, this lock runs only after each full build, so both callers scan storage and create separate indexes. Protect construction with a per-key lock or singleflight so callers wait for and share one build. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. At pkg/storage/unified/search/bleve.go, line 137: <comment>When concurrent requests initialize the same key, this lock runs only after each full build, so both callers scan storage and create separate indexes. Protect construction with a per-key lock or singleflight so callers wait for and share one build.</comment> <file context> @@ -137,7 +134,9 @@ func (b *bleveBackend) BuildIndex(ctx context.Context, return nil, err } + b.cacheMu.Lock() b.cache[key] = idx + b.cacheMu.Unlock() </file context> ``` </details>
pkg/storage/unified/resource/server.go:258
<!-- cubic:v=ddafa05c-b74a-4887-9e52-39cbb7417adc --> <!-- cubic:review-run=c1590f03-341c-591a-b20a-dd2f76bbb7b9 --> <!-- metadata:{"confidence":8} --> P2: When watcher setup fails after lifecycle or search initialization succeeds, `NewResourceServer` returns without cleaning up the initialized components. Stop the lifecycle and cancel all watcher contexts before returning the initialization error. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. At pkg/storage/unified/resource/server.go, line 258: <comment>When watcher setup fails after lifecycle or search initialization succeeds, `NewResourceServer` returns without cleaning up the initialized components. Stop the lifecycle and cancel all watcher contexts before returning the initialization error.</comment> <file context> @@ -255,6 +255,12 @@ func NewResourceServer(opts ResourceServerOptions) (ResourceServer, error) { } } + err := s.Init(ctx) + if err != nil { + s.log.Error("error initializing resource server", "error", err) </file context> ``` </details>
pkg/storage/unified/resource/search.go:216
<!-- cubic:v=75888aeb-3a7a-4462-aac5-6d45dae339d2 --> <!-- cubic:review-run=c1590f03-341c-591a-b20a-dd2f76bbb7b9 --> <!-- metadata:{"confidence":5} --> P2: This new log line calls s.search.TotalDocs() right after the watch goroutine is started. In the bleve backend, TotalDocs() iterates b.cache without holding cacheMu, while a watch event can trigger build -> BuildIndex, which writes to the map under cacheMu.Lock(). A concurrent map read/write causes a fatal Go runtime error and can crash the process. Either move the log (and the TotalDocs call) before the goroutine starts, or better, fix TotalDocs in bleve.go to take b.cacheMu.RLock() so the read is synchronized. <details> <summary>Prompt for AI agents</summary> ```text Check if this issue is valid — if so, understand the root cause and fix it. At pkg/storage/unified/resource/search.go, line 216: <comment>This new log line calls s.search.TotalDocs() right after the watch goroutine is started. In the bleve backend, TotalDocs() iterates b.cache without holding cacheMu, while a watch event can trigger build -> BuildIndex, which writes to the map under cacheMu.Lock(). A concurrent map read/write causes a fatal Go runtime error and can crash the process. Either move the log (and the TotalDocs call) before the goroutine starts, or better, fix TotalDocs in bleve.go to take b.cacheMu.RLock() so the read is synchronized.</comment> <file context> @@ -214,6 +213,7 @@ func (s *searchSupport) init(ctx context.Context) error { }() end := time.Now().Unix() + s.log.Info("search index initialized", "duration_secs", end-start, "total_docs", s.search.TotalDocs()) if IndexMetrics != nil { IndexMetrics.IndexCreationTime.WithLabelValues().Observe(float64(end - start)) </file context> ``` </details>
PR-level
<!-- cubic:review-summary:start --> **3 issues found** across 5 files <!-- cubic:review-summary:end --> <details> <summary>Prompt for AI agents (unresolved issues)</summary> ```text Check if these issues are valid — if so, understand the root cause of each and fix them. If appropriate, use sub-agents to investigate and fix each issue separately. <file name="pkg/storage/unified/search/bleve.go"> <violation number="1" location="pkg/storage/unified/search/bleve.go:137"> P2: When concurrent requests initialize the same key, this lock runs only after each full build, so both callers scan storage and create separate indexes. Protect construction with a per-key lock or singleflight so callers wait for and share one build.</violation> </file> <file name="pkg/storage/unified/resource/server.go"> <violation number="1" location="pkg/storage/unified/resource/server.go:258"> P2: When watcher setup fails after lifecycle or search initialization succeeds, `NewResourceServer` returns without cleaning up the initialized components. Stop the lifecycle and cancel all watcher contexts before returning the initialization error.</violation> </file> <file name="pkg/storage/unified/resource/search.go"> <violation number="1" location="pkg/storage/unified/resource/search.go:216"> P2: This new log line calls s.search.TotalDocs() right after the watch goroutine is started. In the bleve backend, TotalDocs() iterates b.cache without holding cacheMu, while a watch event can trigger build -> BuildIndex, which writes to the map under cacheMu.Lock(). A concurrent map read/write causes a fatal Go runtime error and can crash the process. Either move the log (and the TotalDocs call) before the goroutine starts, or better, fix TotalDocs in bleve.go to take b.cacheMu.RLock() so the read is synchronized.</violation> </file> ``` </details> <sub>Reply with feedback, questions, or to request a fix.<br /><br />[Re-trigger cubic](https://www.cubic.dev/action/re-review/pr/drape-io/crbench-cubic-2026-09/46/ai_pr_review_github_delivery_944be91320b0001de8d3ddb6f6a3e93899271db6bb261e3cff88cec3d9875fd6?returnTo=https%3A%2F%2Fgithub.com%2Fdrape-io%2Fcrbench-cubic-2026-09%2Fpull%2F46)</sub> <!-- cubic:review-post:ai_pr_review_github_delivery_944be91320b0001de8d3ddb6f6a3e93899271db6bb261e3cff88cec3d9875fd6:e532aabce3f9043f37ac7892c05da543b858a1a0:c1590f03-341c-591a-b20a-dd2f76bbb7b9 -->
cursor · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ off by one
d2 medium ✓ exact
valid · matches d1 · impact high
valid · matches d2 · impact medium
raw comments (3)
pkg/storage/unified/search/bleve.go:139
### Index build race on same key **Medium Severity** <!-- DESCRIPTION START --> `BuildIndex` no longer holds `cacheMu` for the full build, but `getOrCreateIndex` still does a non-atomic get-then-build. Concurrent first access to the same `NamespacedResource` can run `bleve.New` on the same directory, overwrite `b.cache`, and drop writes applied to the discarded index. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: bc0a5f01-e1b6-473a-ab23-79c6f16af478 --> <!-- LOCATIONS START pkg/storage/unified/search/bleve.go#L86-L139 pkg/storage/unified/resource/search.go#L286-L306 LOCATIONS END --> <details> <summary>Additional Locations (1)</summary> - [`pkg/storage/unified/resource/search.go#L286-L306`](https://github.com/drape-io/crbench-cursor-2026-09/blob/77306e86e6bc136696ddac82d763f2d1ab21d7f4/pkg/storage/unified/resource/search.go#L286-L306) </details> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90Ojg0ZWZiZGY3LWM0MzAtNDE5ZS04ZDJjLTk1OGE0ZmZmYmVlZiIsImVuY3J5cHRpb25LZXkiOiJSLVp6ZThhVXNCNnBkem4xZTVKNmNjR05lZWRNV0hnRmQzX0Y2VzdZRkFjIiwiYnJhbmNoIjoidDItZ3JhZmFuYS05NzUyOSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90Ojg0ZWZiZGY3LWM0MzAtNDE5ZS04ZDJjLTk1OGE0ZmZmYmVlZiIsImVuY3J5cHRpb25LZXkiOiJSLVp6ZThhVXNCNnBkem4xZTVKNmNjR05lZWRNV0hnRmQzX0Y2VzdZRkFjIiwiYnJhbmNoIjoidDItZ3JhZmFuYS05NzUyOSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjQ3LCJjb21taXRTaGEiOiI3NzMwNmU4NmU2YmMxMzY2OTZkZGFjODJkNzYzZjJkMWFiMjFkN2Y0IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 77306e86e6bc136696ddac82d763f2d1ab21d7f4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
pkg/storage/unified/search/bleve.go:139
### Unlocked index cache map iteration **Medium Severity** <!-- DESCRIPTION START --> `TotalDocs` ranges over `b.cache` without `cacheMu`, while `BuildIndex` now mutates that map from other goroutines. Concurrent map iteration and write is a fatal runtime race, and this window is hit during startup init and Prometheus scrapes. <!-- DESCRIPTION END --> <!-- BUGBOT_BUG_ID: 81299f31-3f27-4d1b-9aa8-b1d439ff7a67 --> <!-- LOCATIONS START pkg/storage/unified/search/bleve.go#L143-L153 pkg/storage/unified/resource/search.go#L215-L216 LOCATIONS END --> <details> <summary>Additional Locations (1)</summary> - [`pkg/storage/unified/resource/search.go#L215-L216`](https://github.com/drape-io/crbench-cursor-2026-09/blob/77306e86e6bc136696ddac82d763f2d1ab21d7f4/pkg/storage/unified/resource/search.go#L215-L216) </details> <div><a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9DVVJTT1IiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjBlZDFkY2Q5LWM0YTYtNDMzYi04MmE5LThmOTVhYjEyZWVhZSIsImVuY3J5cHRpb25LZXkiOiJFbEV0MlpQUnhZa0xJOThSN0l2eTgzY2IyWTdzX1dZSnU5b0lxclo2VkFBIiwiYnJhbmNoIjoidDItZ3JhZmFuYS05NzUyOSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a>&nbsp;<a href="https://cursor.com/agents?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9JTl9XRUIiLCJkYXRhIjp7InJlZGlzS2V5IjoiYnVnYm90OjBlZDFkY2Q5LWM0YTYtNDMzYi04MmE5LThmOTVhYjEyZWVhZSIsImVuY3J5cHRpb25LZXkiOiJFbEV0MlpQUnhZa0xJOThSN0l2eTgzY2IyWTdzX1dZSnU5b0lxclo2VkFBIiwiYnJhbmNoIjoidDItZ3JhZmFuYS05NzUyOSIsInJlcG9Pd25lciI6ImRyYXBlLWlvIiwicmVwb05hbWUiOiJjcmJlbmNoLWN1cnNvci0yMDI2LTA5IiwicHJOdW1iZXIiOjQ3LCJjb21taXRTaGEiOiI3NzMwNmU4NmU2YmMxMzY2OTZkZGFjODJkNzYzZjJkMWFiMjFkN2Y0IiwicHJvdmlkZXIiOiJnaXRodWIifX0" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-web-light.png"><img alt="Fix in Web" width="99" height="28" src="https://cursor.com/assets/images/fix-in-web-dark.png"></picture></a></div> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 77306e86e6bc136696ddac82d763f2d1ab21d7f4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
PR-level
<!-- BUGBOT_REVIEW --> Cursor Bugbot has reviewed your changes using high effort and found 2 potential issues. <!-- BUGBOT_FIX_ALL --> <a href="https://cursor.com/open?link=eyJ2ZXJzaW9uIjoxLCJ0eXBlIjoiQlVHQk9UX0ZJWF9BTExfSU5fQ1VSU09SIiwiZGF0YSI6eyJyZWRpc0tleSI6ImJ1Z2JvdC1tdWx0aTpjNDQ2YzE4Yi05MzVlLTRiNTItYjkxYy1kYTQxMzFlNzFlNWYiLCJlbmNyeXB0aW9uS2V5IjoiYXE5RTVmYmhmY0RrcElDaGxnQzFIUlBmYWUyVjVfMzFHQnNuRC1ZYmxHNCIsImJyYW5jaCI6InQyLWdyYWZhbmEtOTc1MjkiLCJyZXBvT3duZXIiOiJkcmFwZS1pbyIsInJlcG9OYW1lIjoiY3JiZW5jaC1jdXJzb3ItMjAyNi0wOSIsInByb3ZpZGVyIjoiZ2l0aHViIn19" target="_blank" rel="noopener noreferrer"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/fix-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/fix-in-cursor-light.png"><img alt="Fix All in Cursor" width="115" height="28" src="https://cursor.com/assets/images/fix-in-cursor-dark.png"></picture></a> <!-- /BUGBOT_FIX_ALL --> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_BEGIN --> <sup>❌ Bugbot Autofix is OFF. To automatically fix reported issues with cloud agents, enable autofix in the [Cursor dashboard](https://www.cursor.com/dashboard/bugbot).</sup> <!-- BUGBOT_AUTOFIX_REVIEW_FOOTNOTE_END --> <sup>Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 77306e86e6bc136696ddac82d763f2d1ab21d7f4. Configure [here](https://www.cursor.com/dashboard/bugbot).</sup>
Drape (Deep Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ off by one
d2 medium ✓ exact
valid · matches d2 · impact medium
valid · matches d1 · impact high
valid · impact medium
Before the change, every handler called Init, which returns s.initErr after once.Do. Stop() sets initErr, so requests made after Stop used to be rejected. Now that those calls are removed, handlers go straight to the backend even after Stop, which is a real behaviour change. [grounded: diff] [consensus 3/3: valid=3]
raw comments (3)
pkg/storage/unified/resource/search.go:216
**Startup log can crash on concurrent map access** `TotalDocs()` loops over the bleve `cache` map without `cacheMu`. The event goroutine is already running and can write `cache` through `BuildIndex`, and Go aborts on a concurrent map iteration and write. Take `cacheMu.RLock()` in `TotalDocs`. ``` pkg/storage/unified/resource/search.go:216 s.log.Info("search index initialized", "duration_secs", end-start, "total_docs", s.search.TotalDocs()) pkg/storage/unified/search/bleve.go:144 func (b *bleveBackend) TotalDocs() int64 { pkg/storage/unified/search/bleve.go:145 var totalDocs int64 pkg/storage/unified/search/bleve.go:146 for _, v := range b.cache { pkg/storage/unified/search/bleve.go:147 c, err := v.index.DocCount() pkg/storage/unified/search/bleve.go:148 if err != nil { pkg/storage/unified/search/bleve.go:149 continue pkg/storage/unified/search/bleve.go:150 } pkg/storage/unified/search/bleve.go:151 totalDocs += int64(c) pkg/storage/unified/search/bleve.go:152 } ```
pkg/storage/unified/search/bleve.go:88
**Parallel builds of one index lose writes** With the lock covering only the final cache write, a Search and a watch event can build the same key at once. The last build replaces the other, dropping events already written to it. Serialize builds per key (per-key mutex or singleflight) and re-check the cache first. ``` pkg/storage/unified/search/bleve.go:88 (base) b.cacheMu.Lock() pkg/storage/unified/search/bleve.go:89 (base) defer b.cacheMu.Unlock() pkg/storage/unified/search/bleve.go:137 b.cacheMu.Lock() pkg/storage/unified/search/bleve.go:138 b.cache[key] = idx pkg/storage/unified/search/bleve.go:139 b.cacheMu.Unlock() pkg/storage/unified/resource/search.go:287 func (s *searchSupport) getOrCreateIndex(ctx context.Context, key NamespacedResource) (ResourceIndex, error) { pkg/storage/unified/resource/search.go:288 // TODO??? pkg/storage/unified/resource/search.go:289 // We want to block while building the index and return the same index for the key pkg/storage/unified/resource/search.go:290 // simple mutex not great... we don't want to block while anything in building, just the same key pkg/storage/unified/resource/search.go:292 idx, err := s.search.GetIndex(ctx, key) ```
pkg/storage/unified/resource/server.go:449
**Requests still served after Stop()** Stop() sets initErr, but the handlers stopped checking it once the per-request Init calls were removed. Create, Read, Watch and the other handlers now keep calling a stopped backend. Return s.initErr at the top of each handler when it is non-nil. ``` pkg/storage/unified/resource/server.go:449 (base) if err := s.Init(ctx); err != nil { pkg/storage/unified/resource/server.go:450 (base) return nil, err pkg/storage/unified/resource/server.go:451 (base) } pkg/storage/unified/resource/server.go:491 (base) if err := s.Init(ctx); err != nil { pkg/storage/unified/resource/server.go:320 func (s *server) Stop(ctx context.Context) error { pkg/storage/unified/resource/server.go:321 s.initErr = fmt.Errorf("service is stopping") pkg/storage/unified/resource/server.go:323 var stopFailed bool pkg/storage/unified/resource/server.go:324 if s.lifecycle != nil { pkg/storage/unified/resource/server.go:325 err := s.lifecycle.Stop(ctx) pkg/storage/unified/resource/server.go:326 if err != nil { ```
Drape (Review) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ off by one
d2 medium ✓ exact
valid · impact low
Before this change, each handler called s.Init(ctx), which returns s.initErr. Stop sets s.initErr to 'service is stopping' and then 'service is stopped', so requests made after Stop used to be rejected. The change removed every one of these calls, and no handler checks initErr now, so requests are still served after Stop. [grounded: diff] [consensus 3/3: valid=3]
valid · matches d1 · impact high
valid · matches d2 · impact medium
raw comments (3)
pkg/storage/unified/resource/server.go:258
**Requests still served after server Stop** Init now runs once in `NewResourceServer`, and the per-request `s.Init(ctx)` checks were removed from every handler. Since `Stop` sets `initErr` to 'service is stopping', handlers no longer check it. Check `s.initErr` (or a stopping flag) in the handlers, or keep a cheap guard. ``` pkg/storage/unified/resource/server.go:258 err := s.Init(ctx) pkg/storage/unified/resource/server.go:259 if err != nil { pkg/storage/unified/resource/server.go:260 s.log.Error("error initializing resource server", "error", err) pkg/storage/unified/resource/server.go:261 return nil, err ```
pkg/storage/unified/search/bleve.go:88
**Concurrent index builds for same key lose documents** `BuildIndex` no longer holds `cacheMu` during the build, so `GetIndex` no longer waits and finds the cached index. Two callers for an unindexed key each build their own, and the last cache write wins, dropping writes to the other. Dedupe builds per key (singleflight or a per-key lock). ``` pkg/storage/unified/search/bleve.go:88 (base) b.cacheMu.Lock() pkg/storage/unified/search/bleve.go:89 (base) defer b.cacheMu.Unlock() pkg/storage/unified/search/bleve.go:137 b.cacheMu.Lock() pkg/storage/unified/search/bleve.go:138 b.cache[key] = idx ```
pkg/storage/unified/search/bleve.go:146
**TotalDocs reads index cache map without lock** `TotalDocs` iterates `b.cache` with no lock, and `init` now calls it right after starting the event goroutine. That goroutine can call `BuildIndex`, which writes the map, so a concurrent map read and write can crash the process. Take b.cacheMu.RLock() in TotalDocs. ``` pkg/storage/unified/resource/search.go:216 s.log.Info("search index initialized", "duration_secs", end-start, "total_docs", s.search.TotalDocs()) pkg/storage/unified/search/bleve.go:137 b.cacheMu.Lock() pkg/storage/unified/search/bleve.go:138 b.cache[key] = idx pkg/storage/unified/search/bleve.go:139 b.cacheMu.Unlock() pkg/storage/unified/search/bleve.go:144 func (b *bleveBackend) TotalDocs() int64 { pkg/storage/unified/search/bleve.go:145 var totalDocs int64 pkg/storage/unified/search/bleve.go:146 for _, v := range b.cache { pkg/storage/unified/search/bleve.go:147 c, err := v.index.DocCount() pkg/storage/unified/search/bleve.go:148 if err != nil { pkg/storage/unified/search/bleve.go:149 continue ```
greptile · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
d2 medium ✗ —
valid · impact low
Init now runs eagerly in NewResourceServer using the server's own ctx. That ctx comes from context.WithCancel(claims.WithClaims(context.Background(), ...)), and NewResourceServer has no context parameter, so the caller cannot cancel lifecycle.Init or the DB ping. Before this change, Init ran lazily on each request's ctx. [grounded: diff] [consensus 3/3: valid=3]
minor
This is a concern about test coverage and process: the intentional skip has a TODO and doesn't change runtime behavior. That makes it a preference note, not a defect. [grounded: diff] [consensus 3/3: minor=3]
raw comments (3)
pkg/storage/unified/resource/server.go:258
<a href="#"><img alt="P1" src="https://greptile-static-assets.s3.amazonaws.com/badges/p1.svg?v=9" align="top"></a> **Startup Cancellation Is Lost** Resource server construction now runs lifecycle and database initialization with a server-owned context derived from `context.Background()`. If database initialization or `PingContext` blocks, cancellation of the surrounding service startup cannot interrupt it, so startup or shutdown may hang. Pass the caller's startup context into this initialization path, or keep initialization at a lifecycle boundary that has that context.
pkg/server/module_server_test.go:38
<a href="#"><img alt="P2" src="https://greptile-static-assets.s3.amazonaws.com/badges/p2.svg?v=9" align="top"></a> **PostgreSQL Coverage Is Disabled** This skip makes PostgreSQL CI pass without running the only integration test that starts the storage-server module and verifies its metrics endpoint. Because it only notes the existing Drone failure and provides no tracked re-enable condition, PostgreSQL-specific startup regressions can remain hidden. Fix the failure or connect this temporary skip to concrete remediation. Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!
PR-level
<h3>Greptile Summary</h3> This PR moves resource-server initialization into construction, initializes search before the main watcher, propagates tracing contexts through search and SQL operations, narrows Bleve cache locking, standardizes search logging, and skips one PostgreSQL integration test. - Makes backend, search, and watcher initialization synchronous during server construction. - Allows different Bleve indexes to be built without holding the cache mutex for the complete build. - Adds search initialization duration/document-count logging and propagates child span contexts. - Disables the storage-server metrics integration test in PostgreSQL CI. <h3>Confidence Score: 4/5</h3> The PR is not yet safe to merge because eager database initialization cannot be canceled through the service startup context, and PostgreSQL coverage of the affected startup path is disabled. Resource-server construction can remain blocked after its caller cancels because initialization uses a background-derived context, while the new PostgreSQL skip also removes targeted regression coverage for storage-server module startup. **Files Needing Attention:** pkg/storage/unified/resource/server.go, pkg/server/module_server_test.go <details><summary><h3>Important Files Changed</h3></summary> | Filename | Overview | |----------|----------| | pkg/storage/unified/resource/server.go | Moves initialization into the constructor and reorders search initialization before watcher startup; the eager path loses caller-controlled cancellation. | | pkg/storage/unified/resource/search.go | Propagates tracing contexts, standardizes latency logging, and reports index initialization statistics. | | pkg/storage/unified/search/bleve.go | Narrows cache locking to publication of completed indexes, allowing index construction to proceed concurrently. | | pkg/storage/unified/sql/backend.go | Propagates the tracing span context through the resource-statistics transaction. | | pkg/server/module_server_test.go | Skips PostgreSQL execution of the storage-server module metrics integration test. | </details> <!-- greptile_other_comments_section --> <sub>Reviews (1): Last reviewed commit: ["Update pkg/server/module\_server\_test.go ..."](https://github.com/drape-io/crbench-greptile-2026-09/commit/e4b26d1f22f5d04517feebb204da1976e4b88440) | [Re-trigger Greptile](https://app.greptile.com/api/retrigger?id=64284523)</sub>
Mjolnir (Opus 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
d2 medium ✓ exact
valid · matches d2 · impact medium
valid · impact low
Stop() still sets initErr, but every handler's Init check was removed, so nothing reads initErr any more. Before this change, requests made after Stop were rejected with that clear error; now they reach the stopped backend. [grounded: tree pkg/storage/unified/resource/server.go] [consensus 3/3: valid=3]
valid · impact low
The new log call runs after the handleEvent goroutine starts. TotalDocs loops over b.cache without taking cacheMu, while BuildIndex (reached through getOrCreateIndex) writes b.cache. This concurrent map read and write can crash the process, and the change adds this new racing call site. [grounded: tree pkg/storage/unified/resource/search.go] [consensus 3/3: valid=3]
raw comments (3)
pkg/storage/unified/resource/search.go:216
[P2] New TotalDocs() call can race with the event goroutine's map writes The new `s.log.Info(..., s.search.TotalDocs())` runs right after the goroutine that consumes `WatchWriteEvents` has started. `bleveBackend.TotalDocs` ranges over `b.cache` without taking `cacheMu`. Meanwhile `handleEvent` → `getOrCreateIndex` → `BuildIndex` can write `b.cache[key]` at the same moment. That happens whenever a write event arrives for a namespace/resource that wasn't indexed during init, for example one below `initMinSize`. A map write during range iteration can cause Go's fatal "concurrent map iteration and map write" error and crash the server at startup. `TotalDocs` should take `cacheMu.RLock()`, or the log should be emitted before the event goroutine starts.
pkg/storage/unified/resource/server.go:452
[P3] Requests are no longer rejected after Stop() `Stop` still sets `s.initErr = fmt.Errorf("service is stopping")`, but nothing reads that value any more. Before this change, every handler (Create/Update/Delete/Read/List/Watch/Search/etc.) called `s.Init(ctx)`, which returned the stored `initErr`. That rejected requests once shutdown had begun or the lifecycle Stop had failed. With the per-request `Init` checks removed, requests that arrive during or after shutdown go straight to a backend that is stopped or stopping. They fail with backend-specific errors instead of the clear "service is stopping" error. If that gating is still wanted, handlers need a cheap check of the stopped/initErr state.
pkg/storage/unified/resource/search.go:216
[P3] New TotalDocs call can race with cache writes during startup The new startup log line calls `s.search.TotalDocs()`, and `bleveBackend.TotalDocs` reads `b.cache` without taking `cacheMu`. By the time this line runs, the `handleEvent` goroutine started just above it is already reading watch events. An event for a namespace/resource that has no index yet goes through `getOrCreateIndex` → `BuildIndex`, which writes `b.cache[key]` under the lock. TotalDocs ranges over the same map without the lock, so the two can overlap. Go treats a concurrent map read and write as a fatal runtime error, so this would crash the server process. It only happens if a write event for a new resource arrives at the moment search init finishes, so it is unlikely but possible. Taking `cacheMu.RLock()` inside `TotalDocs` would fix this call site and also the metrics collector that calls the same function.
Mjolnir (Sonnet 5.5) · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✗ —
d2 medium ✓ exact
valid · matches d2 · impact medium
raw comments (1)
pkg/storage/unified/resource/search.go:216
[P2] Unsynchronized cache read in the new init log line races with the event goroutine The new `s.log.Info(..., s.search.TotalDocs())` runs right after `init` launches the goroutine that calls `handleEvent`. `bleveBackend.TotalDocs` (pkg/storage/unified/search/bleve.go:144) iterates `b.cache` without taking `cacheMu`. `handleEvent` can reach `getOrCreateIndex`, then `build`, then `BuildIndex`, which writes `b.cache[key]` under the lock. So the iteration can overlap a write to the map if a write event arrives for a namespace or resource that has no index yet (for example, one below `initMinSize`) during startup. That is a Go data race and can end in a "concurrent map iteration and map write" fatal. Before this change `TotalDocs` was only called from the metrics path. Fix by taking `cacheMu.RLock` in `TotalDocs`, or by computing the count before starting the event goroutine.
qodo · cross-file engine perris 0.11.1+proven.tiered0.25.call.read.subclass.type
DefectSeverityFoundGrade
d1 high ✓ exact
d2 medium ✗ —
valid · matches d1 · impact high
valid · impact low
The change adds an eager `s.Init(ctx)` call to `NewResourceServer`. If it fails, the function returns nil without calling `s.cancel()` or `lifecycle.Stop`. Because the change also moved `search.init` ahead of `initWatcher`, the search goroutine (which runs on `context.Background()`) has already started when `initWatcher` fails, and the caller has no server handle to stop it. This is a real leak, but it only happens on a startup failure, which usually ends the process anyway, so the impact is low. [grounded: diff] [consensus 3/3: valid=3]
raw comments (4)
pkg/storage/unified/search/bleve.go:89
<img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> 1\. Concurrent searches can miss updates <code>🐞 Bug</code> <code>≡ Correctness</code> <pre> <b><i>BuildIndex</i></b> no longer holds <b><i>cacheMu</i></b> across index construction, so multiple cache-missing callers can independently build and unconditionally assign indexes for the same key. If an event updates the first cached index while a second build is finishing from an older list snapshot, the second assignment replaces it and the update remains absent from subsequent searches. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description Concurrent cache misses can build the same search index independently, allowing a later stale build to replace an index that already received a watch update. ## Fix Focus Areas - pkg/storage/unified/search/bleve.go[87-140] - pkg/storage/unified/resource/search.go[287-305] ## Recommended Fix Coordinate builds per `NamespacedResource` rather than restoring the global build lock. After acquiring the key-specific lock, recheck the cache and return the existing index when another caller has completed it; only one caller should construct and publish an index for a given key, while builds for different keys remain concurrent. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
pkg/storage/unified/resource/server.go:261
<img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> 2\. Failed startup leaves watchers running <code>🐞 Bug</code> <code>☼ Reliability</code> <pre> <b><i>NewResourceServer</i></b> returns <b><i>nil</i></b> immediately when <b><i>Init</i></b> fails, without stopping lifecycle components or canceling work completed by earlier initialization stages. If <b><i>initWatcher</i></b> fails after <b><i>search.init</i></b> has started its background watcher, the caller has no server handle to stop and that watcher continues on a background context. </pre> <details> <summary><strong>Agent Prompt</strong></summary> ``` ## Issue description Eager initialization can fail after lifecycle components and the search watcher have started, but the constructor returns without canceling or stopping those resources. ## Fix Focus Areas - pkg/storage/unified/resource/server.go[258-262] - pkg/storage/unified/resource/server.go[292-317] - pkg/storage/unified/resource/search.go[201-213] ## Recommended Fix On constructor initialization failure, cancel the server context and stop every lifecycle stage that initialized successfully before returning. Run the search watcher with the server-owned context instead of `context.Background()` so constructor cleanup and normal server shutdown both terminate it. ``` <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> </details>
PR-level
<h3>PR Summary by Qodo</h3> Initialize unified storage before serving requests <code>🐞 Bug fix</code> <code>✨ Enhancement</code> <code>🧪 Tests</code> <code>🕐 20-40 Minutes</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <details> <summary>AI Description</summary> <dl> <dd> <br/> ><pre> >• Initialize resource storage and search indexes before accepting requests. >• Build Bleve indexes concurrently, then start change watching after initial indexing. >• Improve tracing and logging; skip an unstable PostgreSQL instrumentation test. ></pre> </dd> </dl> </details> <details> <summary>Diagram</summary> <dl> <dd> <br/> ```mermaid graph TD A["Server constructor"] --> B["Lifecycle hooks"] --> C["Search support"] --> D["SQL statistics"] --> E["Bleve indexes"] --> F["Change watcher"] --> G["Ready RPC server"] ``` </dd> </dl> </details> <details> <summary>High-Level Assessment</summary> <dl> <dd> <br/> >The following are alternative approaches to this PR: <details> <summary><b>1. Explicit context-aware Start phase</b></summary> <dl> <dd> - ➕ Separates object construction from expensive startup I/O - ➕ Allows callers to apply startup cancellation and deadlines - ➕ Makes lifecycle boundaries explicit - ➖ Requires coordinated changes across all server construction sites - ➖ Callers could accidentally expose an unstarted server without stronger API enforcement </dd> </dl> </details> <details> <summary><b>2. Retain lazy request-time initialization</b></summary> <dl> <dd> - ➕ Preserves fast construction and the existing lifecycle model - ➕ Avoids blocking startup when storage is temporarily unavailable - ➖ Moves initialization latency and failures into the first request - ➖ Complicates readiness guarantees and index-versus-watcher ordering </dd> </dl> </details> >**Recommendation:** Eager constructor initialization is the best scoped fix because it prevents requests from observing a partially initialized server and establishes deterministic index-before-watcher ordering. A context-aware Start or factory API would be cleaner long term, but requires broader call-site and lifecycle changes. </dd> </dl> </details> <details> <summary> Files changed (5) <code> +25 / -57 </code> </summary> <dl> <dd> <br/> <details> <summary>Enhancement (1) <code> +4 / -4 </code></summary> <dl> <dd> <details> <summary>search.go<code>Propagate tracing and improve search-index logging</code> <code>+4/-4</code></summary> <br/> >Propagate tracing and improve search-index logging > ><pre> >• Propagates child span contexts through search initialization and index building. Uses the component logger for latency warnings and records initialization duration and indexed document count. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-31161001a742b1a2ffe022ffd0a922c6e07f30fc383e860355101f670a9c86ef'>pkg/storage/unified/resource/search.go</a> </details> </dd> </dl> </details> <details> <summary>Bug fix (3) <code> +17 / -53 </code></summary> <dl> <dd> <details> <summary>server.go<code>Eagerly initialize the resource server before returning</code> <code>+11/-46</code></summary> <br/> >Eagerly initialize the resource server before returning > ><pre> >• Runs lifecycle hooks and initial search indexing during server construction, then starts the watcher. Removes repeated lazy initialization checks from RPC methods so construction fails immediately when startup cannot complete. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-e4855c422ca8f979c4567e5d12f34ecd3a438a166dc8906ae3edda28bd5e4d20'>pkg/storage/unified/resource/server.go</a> </details> <details> <summary>bleve.go<code>Allow concurrent Bleve index construction</code> <code>+5/-6</code></summary> <br/> >Allow concurrent Bleve index construction > ><pre> >• Narrows cache locking to the final cache insertion rather than holding a global mutex throughout index construction. This allows startup workers to build independent indexes concurrently while preserving synchronized cache writes. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-284e8a32b46a74f9d10caebfb3b5cbe7ae2d44688be503d35b1baf225b5aab3a'>pkg/storage/unified/search/bleve.go</a> </details> <details> <summary>backend.go<code>Propagate SQL statistics trace context</code> <code>+1/-1</code></summary> <br/> >Propagate SQL statistics trace context > ><pre> >• Passes the child span context into resource-statistics operations so downstream work is attached to the correct trace. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-b324afdde67570d986fec2659aedab5860744cbfb64b1edd4eaf59cc3fc49f94'>pkg/storage/unified/sql/backend.go</a> </details> </dd> </dl> </details> <details> <summary>Tests (1) <code> +4 / -0 </code></summary> <dl> <dd> <details> <summary>module_server_test.go<code>Skip unstable PostgreSQL instrumentation integration test</code> <code>+4/-0</code></summary> <br/> >Skip unstable PostgreSQL instrumentation integration test > ><pre> >• Skips the instrumentation-server integration test for PostgreSQL because it is currently unreliable in Drone, while retaining coverage for supported database configurations. ></pre> > ><a href='https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-b54b047f987664fb079a458b6478252515709b7f05bede246f8ae7fa0228b75c'>pkg/server/module_server_test.go</a> </details> </dd> </dl> </details> </dd> </dl> </details>
PR-level
<h3>Code Review by Qodo</h3> <code>🐞 Bugs (2)</code> <code>📘 Rule violations (0)</code> <code>📜 Skill insights (0)</code> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <br/> <img src="https://img.shields.io/badge/High-634FD1?style=flat-square" height="20px" alt="Action required"> <details> <summary> 1. Concurrent searches can miss updates <code>🐞 Bug</code> <code>≡ Correctness</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>BuildIndex</i></b> no longer holds <b><i>cacheMu</i></b> across index construction, so multiple cache-missing callers >can independently build and unconditionally assign indexes for the same key. If an event updates the >first cached index while a second build is finishing from an older list snapshot, the second >assignment replaces it and the update remains absent from subsequent searches. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/storage/unified/search/bleve.go[L88-89]](https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-284e8a32b46a74f9d10caebfb3b5cbe7ae2d44688be503d35b1baf225b5aab3aL88-L89)</code> > >```diff >- b.cacheMu.Lock() >- defer b.cacheMu.Unlock() >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >A search cache miss directly invokes a full build without any per-key coordination, while the >changed backend protects only the final map write. Event handling retrieves and updates whichever >index is currently cached, so another in-flight build can subsequently overwrite that updated index >with its independently populated snapshot. ></pre> > > <code>[pkg/storage/unified/resource/search.go[287-305]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/search.go/#L287-L305)</code> > <code>[pkg/storage/unified/resource/search.go[225-267]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/search.go/#L225-L267)</code> > <code>[pkg/storage/unified/search/bleve.go[96-139]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/search/bleve.go/#L96-L139)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >Concurrent cache misses can build the same search index independently, allowing a later stale build to replace an index that already received a watch update. > >## Fix Focus Areas >- pkg/storage/unified/search/bleve.go[87-140] >- pkg/storage/unified/resource/search.go[287-305] > >## Recommended Fix >Coordinate builds per `NamespacedResource` rather than restoring the global build lock. After acquiring the key-specific lock, recheck the cache and return the existing index when another caller has completed it; only one caller should construct and publish an index for a given key, while builds for different keys remain concurrent. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <br/> <img src="https://img.shields.io/badge/Medium-634FD1?style=flat-square" height="20px" alt="Remediation recommended"> <details> <summary> 2. Failed startup leaves watchers running <code>🐞 Bug</code> <code>☼ Reliability</code></summary> <br/> > <details open> ><summary>Description</summary> ><br/> > ><pre> ><b><i>NewResourceServer</i></b> returns <b><i>nil</i></b> immediately when <b><i>Init</i></b> fails, without stopping lifecycle >components or canceling work completed by earlier initialization stages. If <b><i>initWatcher</i></b> fails >after <b><i>search.init</i></b> has started its background watcher, the caller has no server handle to stop and >that watcher continues on a background context. ></pre> ></details> > <details> ><summary>Code</summary> ><br/> > ><code>[pkg/storage/unified/resource/server.go[R258-261]](https://github.com/drape-io/crbench-qodo-2026-09/pull/45/files#diff-e4855c422ca8f979c4567e5d12f34ecd3a438a166dc8906ae3edda28bd5e4d20R258-R261)</code> > >```diff >+ err := s.Init(ctx) >+ if err != nil { >+ s.log.Error("error initializing resource server", "error", err) >+ return nil, err >``` ></details> > <details> ><summary>Evidence</summary> ><br/> > ><pre> >Initialization runs the lifecycle hook and starts the search watcher before attempting the server >watcher, whose creation can return an error. The new constructor error path performs no cleanup, >while the search watcher explicitly discards the server context and the normal cleanup operations >are only implemented by <b><i>Stop</i></b>. ></pre> > > <code>[pkg/storage/unified/resource/server.go[258-262]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/server.go/#L258-L262)</code> > <code>[pkg/storage/unified/resource/server.go[292-317]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/server.go/#L292-L317)</code> > <code>[pkg/storage/unified/resource/server.go[320-341]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/server.go/#L320-L341)</code> > <code>[pkg/storage/unified/resource/server.go[752-770]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/server.go/#L752-L770)</code> > <code>[pkg/storage/unified/resource/search.go[201-213]](https://github.com/drape-io/crbench-qodo-2026-09/blob/175914c5921ef9176b734f670a97a8eaaa085096/pkg/storage/unified/resource/search.go/#L201-L213)</code> ></details> > <details> ><summary>Agent prompt</summary> ><br/> > >``` >The issue below was found during a code review. Follow the provided context and guidance below and implement a solution > >## Issue description >Eager initialization can fail after lifecycle components and the search watcher have started, but the constructor returns without canceling or stopping those resources. > >## Fix Focus Areas >- pkg/storage/unified/resource/server.go[258-262] >- pkg/storage/unified/resource/server.go[292-317] >- pkg/storage/unified/resource/search.go[201-213] > >## Recommended Fix >On constructor initialization failure, cancel the server context and stop every lifecycle stage that initialized successfully before returning. Run the search watcher with the server-owned context instead of `context.Background()` so constructor cleanup and normal server shutdown both terminate it. >``` > <code>ⓘ Copy this prompt and use it to remediate the issue with your preferred AI generation tools</code> ></details> <hr/> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-context:start --> <details><summary><strong>Context sources</strong></summary> <div>Review mode: <code>⚖️ Balanced</code>: This changes resource-server initialization timing, watcher/search startup ordering, and Bleve cache locking across multiple runtime paths, creating meaningful behavioral and concurrency risk.</div> <!-- qodo-context:end --> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:start --> <details> <summary><strong>Tip of the day</strong></summary> <br/> <pre>💡 Did you know, you can reply &#x27;qodo&#x27; on any finding to push back, ask questions, or dig deeper</pre> <a href="https://docs.qodo.ai/tips-and-tricks">More tips ↗</a> | <a href="https://app.qodo.ai/configurations?tab=display-preferences">Customize Qodo ↗</a> | <a href="https://docs.qodo.ai">Qodo docs ↗</a> </details> <img src="https://www.qodo.ai/wp-content/uploads/2025/11/light-grey-line.svg" height="10%" alt="Grey Divider"> <!-- qodo-daily-tip:end --> <!-- https://github.com/drape-io/crbench-qodo-2026-09/commit/175914c5921ef9176b734f670a97a8eaaa085096 --> <a href="https://www.qodo.ai"><img src="https://www.qodo.ai/wp-content/uploads/2025/03/qodo-logo.svg" width="80" alt="Qodo Logo"></a>