Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 60 additions & 14 deletions include/lance/lance.h
Original file line number Diff line number Diff line change
Expand Up @@ -1695,39 +1695,85 @@ typedef enum {
LANCE_FTS_COVERAGE_INDEX_ONLY = 1,
} LanceFtsCoverageMode;

/** How the analyzed terms of one Match query are combined. */
typedef enum {
/** At least one analyzed term must match. */
LANCE_FTS_MATCH_OPERATOR_OR = 0,
/** Every analyzed term must match. */
LANCE_FTS_MATCH_OPERATOR_AND = 1,
} LanceFtsMatchOperator;

/**
* Prepare an OR Match query context for one column.
*
* @deprecated Use lance_dataset_prepare_fts_match_query() to select the Match
* operator explicitly. This compatibility API is equivalent to
* LANCE_FTS_MATCH_OPERATOR_OR.
*/
LanceFtsQueryContext* lance_dataset_prepare_fts_query(
const LanceDataset* dataset,
const char* column,
const char* query,
uint32_t max_fuzzy_distance,
int32_t coverage_mode
);

/**
* Prepare an immutable, process-local FTS query context for one column.
* Prepare an immutable, process-local Match query context for one column.
*
* Preparation pins the dataset handle's current snapshot, enumerates all
* committed FTS segments for `column`, checks fragment coverage, opens those
* segments, and computes one query-specific global BM25 scorer across their
* indexed documents. The context can then be shared by any number of scanners
* created from the exact same process-local dataset snapshot. It has no
* serialization or cross-process transport format. Reopening the same URI and
* manifest version creates a different identity and cannot reuse the context,
* because storage options and object-store endpoints may differ.
* segments, and prepares one global BM25 scorer across their indexed
* documents. `match_operator` supports both AND and OR.
*
* The context can be shared by scanners created from the exact same
* process-local dataset snapshot. It has no serialization or cross-process
* transport format. Reopening the same URI and manifest version creates a
* different identity and cannot reuse the context because storage options and
* object-store endpoints may differ.
*
* In LANCE_FTS_COVERAGE_INDEX_ONLY mode, unindexed fragments are allowed and
* excluded from both the scorer corpus and query results. In STRICT mode any
* unindexed fragment makes this call fail.
*
* Prepared contexts currently support exact Match queries only.
* `max_fuzzy_distance` must be zero because fuzzy execution requires its
* canonical expanded vocabulary to be prepared together with the scorer.
* This restriction does not apply to lance_scanner_full_text_search().
*
* @param max_fuzzy_distance Must be zero for prepared query contexts.
* @param match_operator Fixed-width LanceFtsMatchOperator discriminant.
* @param max_fuzzy_distance Reserved for prepared fuzzy matching and currently
* must be 0. The parameter is retained so enabling
* canonical cross-segment fuzzy vocabulary injection
* later does not require another C ABI change.
* @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant.
* @return Context handle on success, or NULL on error.
*/
LanceFtsQueryContext* lance_dataset_prepare_fts_query(
LanceFtsQueryContext* lance_dataset_prepare_fts_match_query(
const LanceDataset* dataset,
const char* column,
const char* query,
int32_t match_operator,
uint32_t max_fuzzy_distance,
int32_t coverage_mode
);

/**
* Prepare an immutable, process-local Phrase query context for one column.
*
* The selected FTS index must store token positions. `slop == 0` requires an
* exact phrase; a positive value permits that many intervening positions.
* Dataset identity, coverage, sharing, and segment-scoped execution follow the
* same contract as lance_dataset_prepare_fts_match_query().
*
* @param slop Maximum non-negative number of intervening token positions
* permitted between adjacent phrase terms.
* @param coverage_mode Fixed-width LanceFtsCoverageMode discriminant.
* @return Context handle on success, or NULL on error.
*/
LanceFtsQueryContext* lance_dataset_prepare_fts_phrase_query(
const LanceDataset* dataset,
const char* column,
const char* query,
int32_t slop,
int32_t coverage_mode
);

/**
* Close a context handle. NULL-safe. Scanners that already attached this
* context retain shared ownership and remain valid.
Expand Down
45 changes: 38 additions & 7 deletions include/lance/lance.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,11 @@ enum class FtsCoverageMode : int32_t {
IndexOnly = LANCE_FTS_COVERAGE_INDEX_ONLY,
};

enum class FtsMatchOperator : int32_t {
Or = LANCE_FTS_MATCH_OPERATOR_OR,
And = LANCE_FTS_MATCH_OPERATOR_AND,
};

/// Tunable parameters for Dataset::write. Numeric fields default-out via 0;
/// `data_storage_version` defaults out via `std::nullopt`.
///
Expand Down Expand Up @@ -764,18 +769,44 @@ class Dataset {
/// Create a Scanner builder for this dataset.
Scanner scan() const;

/// Prepare a query-specific global BM25 scorer over the committed FTS
/// segments of this pinned snapshot. IndexOnly permits unindexed fragments;
/// Strict rejects them. Prepared contexts currently require
/// `max_fuzzy_distance == 0`. The context can only be attached to scanners
/// created from this exact process-local dataset snapshot.
/// Compatibility wrapper for an OR Match query.
[[deprecated("Use prepare_fts_match_query() to select the Match operator")]]
FtsQueryContext prepare_fts_query(
const std::string& column,
const std::string& query,
uint32_t max_fuzzy_distance = 0,
FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
auto* context = lance_dataset_prepare_fts_query(
handle_.get(), column.c_str(), query.c_str(), max_fuzzy_distance,
return prepare_fts_match_query(column, query, FtsMatchOperator::Or,
max_fuzzy_distance, coverage_mode);
}

/// Prepare a Match query with a global BM25 scorer. AND and OR are
/// supported. `max_fuzzy_distance` is reserved and currently must be zero;
/// keeping it here avoids another API change when canonical cross-segment
/// fuzzy vocabulary injection becomes available.
FtsQueryContext prepare_fts_match_query(
const std::string& column,
const std::string& query,
FtsMatchOperator match_operator = FtsMatchOperator::Or,
uint32_t max_fuzzy_distance = 0,
FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
auto* context = lance_dataset_prepare_fts_match_query(
handle_.get(), column.c_str(), query.c_str(),
static_cast<int32_t>(match_operator), max_fuzzy_distance,
static_cast<int32_t>(coverage_mode));
if (!context) check_error();
return FtsQueryContext(context);
}

/// Prepare a Phrase query. Its FTS index must store token positions and
/// slop must be non-negative.
FtsQueryContext prepare_fts_phrase_query(
const std::string& column,
const std::string& query,
int32_t slop = 0,
FtsCoverageMode coverage_mode = FtsCoverageMode::Strict) const {
auto* context = lance_dataset_prepare_fts_phrase_query(
handle_.get(), column.c_str(), query.c_str(), slop,
static_cast<int32_t>(coverage_mode));
if (!context) check_error();
return FtsQueryContext(context);
Expand Down
Loading
Loading