feat: add Depot omnisearch

depot/api/api.go

@@ -57,14 +57,12 @@ func InitializeRoutes(router *gin.Engine) {
 
 	router.GET("/stats", GetStats)
 	router.GET("/stats/activity", GetActivityStats)
+	router.GET("/search", OmniSearch)
 	router.GET("/stats/uploaders/:entityID", GetUploaderStats)
 	router.GET("/stats/applications/:clientID", GetApplicationStats)
 	router.GET("/files", ListAttributionFiles)
 	router.GET("/files/search", SearchFiles)
-
-	// Files are addressed by id alone here: the id determines the bucket, so a
-	// consumer stores one reference, and a public file gets a URL it can be
-	// embedded with.
+
 	router.GET("/download/:fileID", DownloadFile)
 
 	router.GET("/storage-backends", ListStorageBackends)
@@ -85,8 +83,6 @@ func InitializeRoutes(router *gin.Engine) {
 	router.PATCH("/buckets/:bucketName/grants/:clientID", UpdateBucketGrant)
 	router.DELETE("/buckets/:bucketName/grants/:clientID", DeleteBucketGrant)
 
-	// File routes an application reaches with its own token, resolved by the
-	// bucket's grants.
 	router.GET("/buckets/:bucketName/files", ListFiles)
 	router.POST("/buckets/:bucketName/files", UploadFile)
 	router.GET("/buckets/:bucketName/files/:id", GetFile)

depot/api/search.go

@@ -0,0 +1,42 @@
+package api
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/gaucho-racing/depot/depot/service"
+	"github.com/gin-gonic/gin"
+)
+
+func OmniSearch(c *gin.Context) {
+	Require(c, RequestTokenExists(c))
+
+	query := strings.TrimSpace(c.Query("q"))
+	if len([]rune(query)) < 2 {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "q must contain at least 2 characters"})
+		return
+	}
+	limit, err := strconv.Atoi(c.DefaultQuery("limit", "30"))
+	if err != nil || limit < 1 || limit > 50 {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "limit must be an integer between 1 and 50"})
+		return
+	}
+	bucketIDs, err := accessibleBucketIDs(c)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	results, err := service.OmniSearch(c.Request.Context(), service.OmniSearchOptions{
+		Query:                 query,
+		BucketIDs:             bucketIDs,
+		IncludeAdminResources: RequestTokenIsAdmin(c),
+		Limit:                 limit,
+	})
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+	c.JSON(http.StatusOK, gin.H{"query": query, "results": results})
+}

depot/database/db.go

@@ -32,6 +32,10 @@ func Init() {
 	}
 
 	logger.SugarLogger.Infoln("Connected to database")
+	if err := db.Exec("CREATE EXTENSION IF NOT EXISTS pg_trgm").Error; err != nil {
+		logger.SugarLogger.Fatalf("failed to enable pg_trgm: %v", err)
+		return
+	}
 
 	// Renames from the terminal → storage backend nomenclature change;
 	// AutoMigrate can't rename, so handle existing dev databases explicitly.
@@ -62,6 +66,20 @@ func Init() {
 		logger.SugarLogger.Fatalf("failed to run database migrations: %v", err)
 		return
 	}
+	searchIndexes := []string{
+		`CREATE INDEX IF NOT EXISTS idx_depot_bucket_search ON depot_bucket USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(name, '') || ' ' || coalesce(description, ''))) gin_trgm_ops)`,
+		`CREATE INDEX IF NOT EXISTS idx_depot_file_search ON depot_file USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(bucket_name, '') || ' ' || coalesce(original_name, '') || ' ' || coalesce(path, '') || ' ' || coalesce(content_type, '') || ' ' || coalesce(storage_backend, '') || ' ' || coalesce(created_by_entity_id, '') || ' ' || coalesce(created_by_client_id, ''))) gin_trgm_ops)`,
+		`CREATE INDEX IF NOT EXISTS idx_depot_storage_backend_search ON depot_storage_backend USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(name, '') || ' ' || coalesce(region, '') || ' ' || coalesce(bucket, '') || ' ' || coalesce(endpoint, ''))) gin_trgm_ops)`,
+		`CREATE INDEX IF NOT EXISTS idx_depot_bucket_grant_search ON depot_bucket_grant USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(bucket_name, '') || ' ' || coalesce(client_id, '') || ' ' || coalesce(description, ''))) gin_trgm_ops)`,
+		`CREATE INDEX IF NOT EXISTS idx_depot_file_replica_search ON depot_file_replica USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(file_id, '') || ' ' || coalesce(storage_backend, '') || ' ' || coalesce(error, ''))) gin_trgm_ops)`,
+		`CREATE INDEX IF NOT EXISTS idx_depot_access_log_search ON depot_access_log USING GIN ((lower(coalesce(id, '') || ' ' || coalesce(file_id, '') || ' ' || coalesce(file_name, '') || ' ' || coalesce(bucket_name, '') || ' ' || coalesce(entity_id, '') || ' ' || coalesce(client_id, ''))) gin_trgm_ops)`,
+	}
+	for _, statement := range searchIndexes {
+		if err := db.Exec(statement).Error; err != nil {
+			logger.SugarLogger.Fatalf("failed to create search index: %v", err)
+			return
+		}
+	}
 	logger.SugarLogger.Infoln("AutoMigration complete")
 	DB = db
 }

depot/service/identity.go

@@ -11,6 +11,7 @@ import (
 )
 
 const identityCacheTTL = 5 * time.Minute
+const identityResolveBatchSize = 100
 
 type cachedIdentity struct {
 	summary   sentinel.IdentitySummary
@@ -41,16 +42,19 @@ func ResolveIdentities(ctx context.Context, entityIDs []string) ([]sentinel.Iden
 	}
 
 	if len(missing) > 0 {
-		resolved, err := sentinel.ResolveIdentities(ctx, missing)
-		if err != nil {
-			if summaries, complete := cachedIdentityResults(entityIDs); complete {
-				logger.SugarLogger.Warnf("serving stale identity summaries, refresh failed: %v", err)
-				return summaries, nil
+		for start := 0; start < len(missing); start += identityResolveBatchSize {
+			end := min(start+identityResolveBatchSize, len(missing))
+			resolved, err := sentinel.ResolveIdentities(ctx, missing[start:end])
+			if err != nil {
+				if summaries, complete := cachedIdentityResults(entityIDs); complete {
+					logger.SugarLogger.Warnf("serving stale identity summaries, refresh failed: %v", err)
+					return summaries, nil
+				}
+				return nil, err
+			}
+			for _, summary := range resolved {
+				cachedIdentities[summary.ID] = cachedIdentity{summary: summary, fetchedAt: now}
 			}
-			return nil, err
-		}
-		for _, summary := range resolved {
-			cachedIdentities[summary.ID] = cachedIdentity{summary: summary, fetchedAt: now}
 		}
 	}