diff --git a/_build_scripts/link-validator.js b/_build_scripts/link-validator.js index 2b3c8715b..eacdddacb 100644 --- a/_build_scripts/link-validator.js +++ b/_build_scripts/link-validator.js @@ -45,6 +45,7 @@ const domainsToIgnore = [ 'medium.com', // TODO[g-despot]: started throwing Forbidden 403 (incl. subdomains, e.g. *.medium.com) 'https://www.npmjs.com', 'https://openai.com', + 'https://platform.deepseek.com', // 403s automated requests (site loads fine in a browser) 'https://platform.openai.com', 'https://www.researchgate.net', 'https://simple/', diff --git a/_includes/code/csharp/ModelProvidersTest.cs b/_includes/code/csharp/ModelProvidersTest.cs index 8719c6eb9..f34314fc6 100644 --- a/_includes/code/csharp/ModelProvidersTest.cs +++ b/_includes/code/csharp/ModelProvidersTest.cs @@ -154,6 +154,95 @@ await client.Collections.Create( await client.Collections.Delete("DemoCollection"); } + [Fact(Skip = "Requires MORPH_APIKEY, not configured in CI")] + public async Task TestMorphInstantiation() + { + // START MorphInstantiation + // Best practice: store your credentials in environment variables + string weaviateUrl = Environment.GetEnvironmentVariable("WEAVIATE_URL"); + string weaviateApiKey = Environment.GetEnvironmentVariable("WEAVIATE_API_KEY"); + string morphApiKey = Environment.GetEnvironmentVariable("MORPH_APIKEY"); + + // highlight-start + // Morph requests are built by Weaviate's OpenAI-compatible client, + // so the Morph key is supplied under the OpenAI header name. + using var client = await Connect.Cloud( + weaviateUrl, + weaviateApiKey, + headers: new Dictionary + { + ["X-Openai-Api-Key"] = morphApiKey, + } + ); + + var meta = await client.GetMeta(); + Console.WriteLine(meta.Version); + // highlight-end + // END MorphInstantiation + } + + [Fact(Skip = "Requires MORPH_APIKEY, not configured in CI")] + public async Task TestMorphVectorizer() + { + if (await client.Collections.Exists("DemoCollection")) + await client.Collections.Delete("DemoCollection"); + + // START BasicVectorizerMorph + await client.Collections.Create( + new CollectionCreateParams + { + Name = "DemoCollection", + VectorConfig = new VectorConfigList + { + Configure.Vector( + "title_vector", + v => v.Text2VecMorph(), + sourceProperties: ["title"] + ), + }, + Properties = [Property.Text("title"), Property.Text("description")], + } + ); + // END BasicVectorizerMorph + + var config = await client.Collections.Export("DemoCollection"); + Assert.True(config.VectorConfig.ContainsKey("title_vector")); + Assert.Equal("text2vec-morph", config.VectorConfig["title_vector"].Vectorizer.Identifier); + + await client.Collections.Delete("DemoCollection"); + } + + [Fact(Skip = "Requires MORPH_APIKEY, not configured in CI")] + public async Task TestMorphVectorizerFull() + { + if (await client.Collections.Exists("DemoCollection")) + await client.Collections.Delete("DemoCollection"); + + // START FullVectorizerMorph + await client.Collections.Create( + new CollectionCreateParams + { + Name = "DemoCollection", + VectorConfig = new VectorConfigList + { + Configure.Vector( + "title_vector", + v => v.Text2VecMorph(model: "morph-embedding-v3"), + sourceProperties: ["title"] + ), + }, + Properties = [Property.Text("title"), Property.Text("description")], + } + ); + // END FullVectorizerMorph + + var config = await client.Collections.Export("DemoCollection"); + Assert.True(config.VectorConfig.ContainsKey("title_vector")); + Assert.Equal("text2vec-morph", config.VectorConfig["title_vector"].Vectorizer.Identifier); + + await client.Collections.Delete("DemoCollection"); + } + [Fact] public async Task TestWeaviateVectorizerModel() { diff --git a/_includes/code/howto/configure.backups.py b/_includes/code/howto/configure.backups.py index f143c91d7..e416fb5ba 100644 --- a/_includes/code/howto/configure.backups.py +++ b/_includes/code/howto/configure.backups.py @@ -191,6 +191,24 @@ assert client.collections.exists("Article") assert client.collections.exists("Publication") +# START ListBackups +backups = client.backup.list_backups( + backend="filesystem", + sort_by_starting_time_asc=True, +) + +for backup in backups: + print(backup.backup_id, backup.status, backup.incremental_base_backup_id) +# END ListBackups + +# Test +backup_ids = [backup.backup_id for backup in backups] +assert {"base-backup", "incremental-backup-1", "incremental-backup-2"}.issubset(backup_ids) +bases = {backup.backup_id: backup.incremental_base_backup_id for backup in backups} +assert bases["base-backup"] is None +assert bases["incremental-backup-1"] == "base-backup" +assert bases["incremental-backup-2"] == "incremental-backup-1" + # Clean up client.collections.delete(["Article", "Publication"]) diff --git a/_includes/code/howto/manage-data.create.py b/_includes/code/howto/manage-data.create.py index b758b4cec..6204c8d6c 100644 --- a/_includes/code/howto/manage-data.create.py +++ b/_includes/code/howto/manage-data.create.py @@ -182,6 +182,10 @@ # WithGeoCoordinates # ======================================== +# Test scaffolding; not rendered in the docs. Start from an empty collection so +# the exact count asserted below cannot pick up an object left by an earlier run. +client.collections.delete("Publication") + # START WithGeoCoordinates publications = client.collections.use("Publication") @@ -196,23 +200,34 @@ # END WithGeoCoordinates # TEST - Confirm insert & delete object +# Test scaffolding below; not rendered in the docs. +# A geo property is backed by its own index, and that index is filled in the +# background when the server runs with ASYNC_INDEXING enabled (the test +# instance does), so a geo filter does not see the object the moment insert +# returns. Wait (bounded) for the write to become visible, so the assertion +# tests the geo range and not index visibility. The assertion stays exact. +import time from weaviate.classes.data import GeoCoordinate from weaviate.classes.query import Filter -response = publications.query.fetch_objects( - filters=( - Filter - .by_property("headquartersGeoLocation") - .within_geo_range( - coordinate=GeoCoordinate( - latitude=52.39, - longitude=4.84 - ), - distance=1000 # In meters - ) +geo_range_filter = ( + Filter + .by_property("headquartersGeoLocation") + .within_geo_range( + coordinate=GeoCoordinate( + latitude=52.39, + longitude=4.84 + ), + distance=1000 # In meters ) ) +for _ in range(30): + response = publications.query.fetch_objects(filters=geo_range_filter) + if len(response.objects) >= 1: + break + time.sleep(1) + assert len(response.objects) == 1 obj_uuid = response.objects[0].uuid publications.data.delete_by_id(obj_uuid) diff --git a/_includes/code/howto/search.bm25.gql.py b/_includes/code/howto/search.bm25.gql.py index 18a69fd0a..328170f0b 100644 --- a/_includes/code/howto/search.bm25.gql.py +++ b/_includes/code/howto/search.bm25.gql.py @@ -72,4 +72,29 @@ gqlresponse = client.graphql_raw_query(gql_query) +gql_query = """ +# START BM25OperatorCrossPropertyAnd +{ + Get { + JeopardyQuestion( + limit: 3 + bm25: { + query: "Australian mammal cute" + # highlight-start + searchOperator: { + operator: AndCross, + } + # highlight-end + } + ) { + question + answer + } + } +} +# END BM25OperatorCrossPropertyAnd +""" + +gqlresponse = client.graphql_raw_query(gql_query) + client.close() diff --git a/_includes/code/howto/search.bm25.py b/_includes/code/howto/search.bm25.py index 696fae0b3..36ad62f4d 100644 --- a/_includes/code/howto/search.bm25.py +++ b/_includes/code/howto/search.bm25.py @@ -107,6 +107,36 @@ # End test +# ============================ +# ===== BM25 w/ AND_CROSS ===== +# ============================ + +# START BM25OperatorCrossPropertyAnd +# highlight-start +from weaviate.classes.query import BM25Operator +# highlight-end + +jeopardy = client.collections.use("JeopardyQuestion") +response = jeopardy.query.bm25( + # highlight-start + query="African desert wind", + # Each token must be matched by at least one searched property, + # but not necessarily all by the same property + operator=BM25Operator.and_cross(), + # highlight-end + limit=3, +) + +for o in response.objects: + print(o.properties) +# END BM25OperatorCrossPropertyAnd + + +# Tests +assert response.objects[0].collection == "JeopardyQuestion" +# End test + + # ================================================ # ===== BM25 Query with score / explainScore ===== # ================================================ diff --git a/_includes/code/howto/search.similarity.mmr.py b/_includes/code/howto/search.similarity.mmr.py index 30811ab89..feb486f2b 100644 --- a/_includes/code/howto/search.similarity.mmr.py +++ b/_includes/code/howto/search.similarity.mmr.py @@ -126,6 +126,83 @@ relevant_questions = [o.properties["question"] for o in response_relevant.objects] assert diverse_questions != relevant_questions, "Pure diversity and pure relevance should differ" +# START MMRHybridExample +from weaviate.classes.query import Diversity + +collection = client.collections.get("MMRDemo") + +# Fuse the keyword and vector results into 20 candidates, then select 5 diverse results +response = collection.query.hybrid( + query="Question", + vector=base_vec, + limit=20, + # highlight-start + diversity_selection=Diversity.mmr( + limit=5, + balance=0.5, + ), + # highlight-end +) + +for o in response.objects: + print(o.properties["question"]) +# END MMRHybridExample + +# Test +assert len(response.objects) == 5 +hybrid_mmr_questions = [o.properties["question"] for o in response.objects] + +hybrid_standard = collection.query.hybrid( + query="Question", + vector=base_vec, + limit=5, +) +hybrid_standard_questions = [o.properties["question"] for o in hybrid_standard.objects] +assert set(hybrid_standard_questions) != set( + hybrid_mmr_questions +), "MMR should select different items than standard hybrid search" + +# START MMRPagination +from weaviate.classes.query import Diversity + +collection = client.collections.get("MMRDemo") + +# The query limit is the size of the window that gets diversified. +# The diversity limit is the page size. +query_limit = 10 +page_size = 3 + +# Advance offset by the query limit, NOT by the number of returned objects +for offset in range(0, 30, query_limit): + page = collection.query.near_vector( + near_vector=base_vec, + limit=query_limit, + # highlight-start + offset=offset, + diversity_selection=Diversity.mmr(limit=page_size, balance=0.5), + # highlight-end + ) + + for o in page.objects: + print(offset, o.properties["question"]) +# END MMRPagination + +# Test +paged_questions = [] +for offset in range(0, 30, query_limit): + page = collection.query.near_vector( + near_vector=base_vec, + limit=query_limit, + offset=offset, + diversity_selection=Diversity.mmr(limit=page_size, balance=0.5), + ) + paged_questions += [o.properties["question"] for o in page.objects] + +assert len(paged_questions) == 9 +assert len(set(paged_questions)) == len( + paged_questions +), "Advancing offset by the query limit must not repeat objects across pages" + # Cleanup client.collections.delete("MMRDemo") client.close() diff --git a/_includes/code/java-v6/src/test/java/ModelProvidersTest.java b/_includes/code/java-v6/src/test/java/ModelProvidersTest.java index e3c7aa476..e4cd99c14 100644 --- a/_includes/code/java-v6/src/test/java/ModelProvidersTest.java +++ b/_includes/code/java-v6/src/test/java/ModelProvidersTest.java @@ -119,6 +119,71 @@ void testDigitalOceanVectorizer() throws IOException { client.collections.delete("DemoCollection"); } + @Test + @Disabled("Requires MORPH_APIKEY, not configured in CI") + void testMorphInstantiation() throws Exception { + // START MorphInstantiation + // Best practice: store your credentials in environment variables + String weaviateUrl = System.getenv("WEAVIATE_URL"); + String weaviateApiKey = System.getenv("WEAVIATE_API_KEY"); + String morphApiKey = System.getenv("MORPH_APIKEY"); + + // highlight-start + // Morph requests are built by Weaviate's OpenAI-compatible client, + // so the Morph key is supplied under the OpenAI header name. + WeaviateClient client = WeaviateClient.connectToWeaviateCloud( + weaviateUrl, + weaviateApiKey, + config -> config.setHeaders(Map.of("X-Openai-Api-Key", morphApiKey))); + + System.out.println(client.isReady()); // Should print: `True` + // highlight-end + + client.close(); // Free up resources + // END MorphInstantiation + } + + @Test + @Disabled("Requires MORPH_APIKEY, not configured in CI") + void testMorphVectorizer() throws IOException { + client.collections.delete("DemoCollection"); + // START BasicVectorizerMorph + client.collections.create("DemoCollection", + col -> col + .vectorConfig( + VectorConfig.text2vecMorph("title_vector", c -> c.sourceProperties("title"))) + .properties(Property.text("title"), Property.text("description"))); + // END BasicVectorizerMorph + + var config = client.collections.getConfig("DemoCollection").get(); + assertThat(config.vectors()).containsKey("title_vector"); + assertThat(config.vectors().get("title_vector").getClass().getSimpleName()) + .isEqualTo("Text2VecMorphVectorizer"); + client.collections.delete("DemoCollection"); + } + + @Test + @Disabled("Requires MORPH_APIKEY, not configured in CI") + void testMorphVectorizerFull() throws IOException { + client.collections.delete("DemoCollection"); + // START FullVectorizerMorph + client.collections.create("DemoCollection", + col -> col + .vectorConfig(VectorConfig.text2vecMorph("title_vector", + c -> c.sourceProperties("title") + .model("morph-embedding-v3") + .baseUrl("https://api.morphllm.com") // Base URL; an existing path is preserved + .endpoint("/v1/embeddings"))) // Path appended to the base URL + .properties(Property.text("title"), Property.text("description"))); + // END FullVectorizerMorph + + var config = client.collections.getConfig("DemoCollection").get(); + assertThat(config.vectors()).containsKey("title_vector"); + assertThat(config.vectors().get("title_vector").getClass().getSimpleName()) + .isEqualTo("Text2VecMorphVectorizer"); + client.collections.delete("DemoCollection"); + } + @Test void testWeaviateVectorizerModel() throws IOException { // START VectorizerWeaviateCustomModel diff --git a/_includes/feature-notes/boost.mdx b/_includes/feature-notes/boost.mdx index 449fd66d6..b522d781e 100644 --- a/_includes/feature-notes/boost.mdx +++ b/_includes/feature-notes/boost.mdx @@ -1,3 +1,2 @@ -:::caution Preview: added in `v1.38` -This is a preview feature. The API may change in future releases. +:::info Added in `v1.39` ::: diff --git a/_includes/feature-notes/hnsw-snapshots.mdx b/_includes/feature-notes/hnsw-snapshots.mdx index ff9190c52..4d7fb6b8b 100644 --- a/_includes/feature-notes/hnsw-snapshots.mdx +++ b/_includes/feature-notes/hnsw-snapshots.mdx @@ -1,2 +1,5 @@ -:::info Added in `v1.31` +:::info Added in `v1.31` · Changed in `v1.39` + +Starting in `v1.39`, HNSW snapshots are created and managed automatically, and are no longer configurable. + ::: diff --git a/_includes/feature-notes/v137-preview.mdx b/_includes/feature-notes/v137-preview.mdx index 156891a60..eeafa5aa2 100644 --- a/_includes/feature-notes/v137-preview.mdx +++ b/_includes/feature-notes/v137-preview.mdx @@ -1,7 +1,6 @@ -:::caution Preview — added in `v1.37` +:::info Added in `v1.37.3` and `v1.38.6` -This is a preview feature. The API may change in future releases. - -- **Multi-node clusters**: MMR reranking may produce suboptimal results for collections whose shards are distributed across multiple nodes, since each shard returns its own candidate set before the coordinator reranks them. We are actively working on improving this. +**Diversity selection (MMR)** for vector search was added in **`v1.37.3`**.
+**Diversity selection (MMR)** for hybrid search was added in **`v1.38.6`**. ::: diff --git a/_includes/release-history.md b/_includes/release-history.md index d0443dab5..ba65becc6 100644 --- a/_includes/release-history.md +++ b/_includes/release-history.md @@ -2,17 +2,18 @@ This table lists recent Weaviate Database versions and corresponding client libr | Weaviate Database
([GitHub][cWeaviate]) | First
release date | Python
([GitHub][cPython]) | TypeScript/
JavaScript
([GitHub][cTypeScript]) | Go
([GitHub][cGo]) | Java
([GitHub][cJava]) | C#
([GitHub][cCSharp]) | | :------------------------------------------------------------------ | :---------------------- | :-------------------------------------------------------------------------------: | :--------------------------------------------------------------------------: | :-------------------------------------------------------------------------: | :-----------------------------------------------------------------: | :-----------------------------------------------------------------------------: | -| [1.38.x](https://github.com/weaviate/weaviate/releases/tag/v1.38.0) | 2026-06-05 | [4.22.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.22.0) | - | - | [6.3.0](https://github.com/weaviate/java-client/releases/tag/6.3.0) | - | +| [1.39.x](https://github.com/weaviate/weaviate/releases/tag/v1.39.0) | 2026-08-04 | [4.23.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.23.0) | - | - | [6.3.1](https://github.com/weaviate/java-client/releases/tag/6.3.1) | - | +| [1.38.x](https://github.com/weaviate/weaviate/releases/tag/v1.38.0) | 2026-06-05 | [4.22.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.22.0) | [3.14.x](https://github.com/weaviate/typescript-client/releases/tag/v3.14.0) | - | [6.3.0](https://github.com/weaviate/java-client/releases/tag/6.3.0) | - | | [1.37.x](https://github.com/weaviate/weaviate/releases/tag/v1.37.0) | 2026-04-16 | [4.21.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.21.0) | [3.13.x](https://github.com/weaviate/typescript-client/releases/tag/v3.13.0) | [5.7.3](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.7.3) | [6.2.0](https://github.com/weaviate/java-client/releases/tag/6.2.0) | N/A | | [1.36.x](https://github.com/weaviate/weaviate/releases/tag/v1.36.0) | 2026-02-24 | [4.20.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.20.0) | [3.12.x](https://github.com/weaviate/typescript-client/releases/tag/v3.12.0) | [5.7.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.7.0) | [6.1.0](https://github.com/weaviate/java-client/releases/tag/6.1.0) | [1.0.1](https://github.com/weaviate/weaviate-dotnet-client/releases/tag/v1.0.1) | | [1.35.x](https://github.com/weaviate/weaviate/releases/tag/v1.35.0) | 2025-12-17 | [4.19.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.19.0) | [3.11.x](https://github.com/weaviate/typescript-client/releases/tag/v3.11.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | [1.0.0](https://github.com/weaviate/weaviate-dotnet-client/releases/tag/v1.0.0) | -| [1.34.x](https://github.com/weaviate/weaviate/releases/tag/v1.34.0) | 2025-11-05 | [4.18.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.18.0) | [3.10.x](https://github.com/weaviate/typescript-client/releases/tag/v3.10.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | - |
Older releases | Weaviate Database
([GitHub][cWeaviate]) | First
release date | Python
([GitHub][cPython]) | TypeScript/
JavaScript
([GitHub][cTypeScript]) | Go
([GitHub][cGo]) | Java
([GitHub][cJava]) | | :------------------------------------------------------------------ | :---------------------- | :-------------------------------------------------------------------------------: | :------------------------------------------------------------------------: | :---------------------------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------: | +| [1.34.x](https://github.com/weaviate/weaviate/releases/tag/v1.34.0) | 2025-11-05 | [4.18.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.18.0) | [3.10.x](https://github.com/weaviate/typescript-client/releases/tag/v3.10.0) | [5.6.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.6.0) | [6.0.0](https://github.com/weaviate/java-client/releases/tag/6.0.0) | | [1.33.x](https://github.com/weaviate/weaviate/releases/tag/v1.33.0) | 2025-09-25 | [4.17.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.17.0) | [3.9.x](https://github.com/weaviate/typescript-client/releases/tag/v3.9.0) | [5.5.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.5.0) | [5.5.x](https://github.com/weaviate/java-client/releases/tag/5.5.0) | | [1.32.x](https://github.com/weaviate/weaviate/releases/tag/v1.32.0) | 2025-07-14 | [4.16.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.16.0) | [3.8.x](https://github.com/weaviate/typescript-client/releases/tag/v3.8.0) | [5.3.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.3.0) | [5.4.x](https://github.com/weaviate/java-client/releases/tag/5.4.0) | | [1.31.x](https://github.com/weaviate/weaviate/releases/tag/v1.31.0) | 2025-05-30 | [4.15.x](https://github.com/weaviate/weaviate-python-client/releases/tag/v4.15.0) | [3.6.x](https://github.com/weaviate/typescript-client/releases/tag/v3.6.0) | [5.2.x](https://github.com/weaviate/weaviate-go-client/releases/tag/v5.2.0) | [5.3.x](https://github.com/weaviate/java-client/releases/tag/5.3.0) | diff --git a/docs/cloud/manage-clusters/connect.mdx b/docs/cloud/manage-clusters/connect.mdx index 4d0f81c46..64126573e 100644 --- a/docs/cloud/manage-clusters/connect.mdx +++ b/docs/cloud/manage-clusters/connect.mdx @@ -78,6 +78,8 @@ If you don't have an existing API key, you'll need to create one. Follow these s :::note REST Endpoint vs gRPC Endpoint When using an official Weaviate [client library](/weaviate/client-libraries), you need to authenticate using the `REST Endpoint` and your API key. The client will infer the gRPC endpoint automatically and use the more performant gRPC protocol when available. + +To reach the [gRPC-Web interface](/weaviate/api/grpc.md#grpc-web), use the `REST Endpoint` URL, not the `gRPC Endpoint` URL, because gRPC-Web is served on the REST port. ::: ### Environment variables diff --git a/docs/deploy/configuration/backups.md b/docs/deploy/configuration/backups.md index 784983d5f..419bacbe2 100644 --- a/docs/deploy/configuration/backups.md +++ b/docs/deploy/configuration/backups.md @@ -522,6 +522,77 @@ Base backups (and any intermediate incremental backups in a chain) must remain a ::: +### List Backups + +You can list the backups that are stored in a backup backend. The listing reports the status of each backup, the collections it holds, its size, and, for an incremental backup, the backup it was built on. This is how you inspect an existing [chain of incremental backups](#chained-incremental-backups) and confirm that every backup the chain depends on is still present. + +```js +GET /v1/backups/{backend} +``` + +#### Parameters + +##### URL Parameters + +| Name | Type | Required | Description | +| ---- | ---- | ---- | ---- | +| `backend` | string | yes | The name of the backup provider module without the `backup-` prefix, for example `s3`, `gcs`, or `filesystem`. | + +##### Query Parameters + +| Name | Type | Required | Default | Description | +| ---- | ---- | ---- | ---- | ---- | +| `order` | string | no | `desc` | Sort the returned backups by start time, either `asc` (oldest first) or `desc` (newest first). | + +##### Response fields + +| name | type | description | +| ---- | ---- | ---- | +| `id` | string | The identifier of the backup. | +| `classes` | array | The collections the backup contains. | +| `status` | string | The status of the backup, such as `SUCCESS` or `FAILED`. | +| `startedAt` | date | When the backup started. | +| `completedAt` | date | When the backup finished, successfully or not. | +| `size` | number | The size of the backup in GiB, measured before compression. | +| `incremental_base_backup_id` | string | The identifier of the backup that this [incremental backup](#incremental-backups) was built on. Empty when the backup is a full backup. Only returned to root users, see below. Introduced in Weaviate `v1.37.6`. | + +The listing only includes backups whose collections you are authorized to read. Backups you have no read access to are left out rather than causing an error. + +:::caution One name, two different things + +`incremental_base_backup_id` appears on both sides of the backup API, and the two are not interchangeable: + +- On the **create** side it is an input that you supply. It names the backup that the new backup should build on, as described under [Create an incremental backup](#create-an-incremental-backup). +- On the **list** side it is a read-only output. It reports the backup that an already-created backup was built on, which is what lets you walk a chain back to its full base backup. + +You cannot choose a base backup through the list API. The value it returns reflects a decision that was made when that backup was created. + +::: + +:::info The base backup identifier is only returned to root users + +The list-side `incremental_base_backup_id` is treated as sensitive. Weaviate only fills it in when it has confirmed that the caller is a [root user](/docs/deploy/configuration/configuring-rbac.md). Any other caller receives an empty value for this field, even one with full backup permissions and even when the backup really is incremental. If you are auditing a backup chain and every entry comes back empty, check the identity you are connecting with before concluding that no incremental backups exist. + +::: + + + +
+ Code output + +``` +base-backup BackupStatus.SUCCESS None +incremental-backup-1 BackupStatus.SUCCESS base-backup +incremental-backup-2 BackupStatus.SUCCESS incremental-backup-1 +``` + +
+ ### Cancel Backup An ongoing backup can be cancelled at any time. The backup process will be stopped, and the backup will be marked as `CANCELLED`. diff --git a/docs/deploy/configuration/env-vars/index.md b/docs/deploy/configuration/env-vars/index.md index c09da6ecf..60026acc0 100644 --- a/docs/deploy/configuration/env-vars/index.md +++ b/docs/deploy/configuration/env-vars/index.md @@ -86,11 +86,11 @@ import APITable from '@site/src/components/APITable'; | `OPERATIONAL_MODE` | Sets the [mode of operation](../status.md#operational-modes) for the instance, limiting the available operations based on the mode selected. Options: `ReadWrite` (default), `ReadOnly`, `WriteOnly`, `ScaleOut`. These values are case-sensitive and matched exactly. An unrecognized value (such as `READ_ONLY`) silently falls back to the default `ReadWrite`, without a warning or an error, so a node you intended to restrict stays fully writable. | `string` | `ReadWrite` | | `ORIGIN` | Set the http(s) origin for Weaviate | `string - HTTP origin` | `https://my-weaviate-deployment.com` | | `PERSISTENCE_DATA_PATH` | Path to the Weaviate data store.
[Note about file systems and performance](/weaviate/concepts/resources.md#file-system). | `string - file path` | `/var/lib/weaviate`
Defaults to `./data`| -| `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` | If set, [HNSW snapshotting](/weaviate/concepts/storage.md#persistence-and-crash-recovery) will be disabled. Default: `false` (enabled) as of `v1.36`; `true` (disabled) in `v1.31`–`v1.35`
Added in `v1.31` | `boolean` | `false` | -| `PERSISTENCE_HNSW_SNAPSHOT_INTERVAL_SECONDS` | The minimum time in seconds that must pass before the next snapshot is created. Default: `21600` seconds (6 hours)
Added in `v1.31` | `string - number` | `3600` | -| `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_NUMBER` | The minimum number of new commit log files created since the last snapshot. Default: `1`
Added in `v1.31` | `string - number` | `100` | -| `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_SIZE_PERCENTAGE` | The minimum total size of new commit logs (as a percentage of the previous snapshot's size) required to trigger a new snapshot. Default: `5` (meaning 5% of the previous snapshot's size in new commit logs)
Added in `v1.31` | `string - number` | `15` | -| `PERSISTENCE_HNSW_SNAPSHOT_ON_STARTUP` | If set, Weaviate will try to create a new snapshot during startup if there are changes in the commit log since the last snapshot. If there are no changes, then the existing snapshot will be loaded. Default: `true`
Added in `v1.31` | `boolean` | `false` | +| `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: if set, HNSW snapshotting is disabled. Default: `false` (enabled) as of `v1.36`; `true` (disabled) in `v1.31` through `v1.35`. See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `boolean` | `false` | +| `PERSISTENCE_HNSW_SNAPSHOT_INTERVAL_SECONDS` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: the minimum time in seconds that must pass before the next snapshot is created. Default: `21600` seconds (6 hours). See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `string - number` | `3600` | +| `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_NUMBER` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: the minimum number of new commit log files created since the last snapshot. Default: `1`. See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `string - number` | `100` | +| `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_SIZE_PERCENTAGE` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: the minimum total size of new commit logs (as a percentage of the previous snapshot's size) required to trigger a new snapshot. Default: `5` (meaning 5% of the previous snapshot's size in new commit logs). See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `string - number` | `15` | +| `PERSISTENCE_HNSW_SNAPSHOT_ON_STARTUP` | **Deprecated in `v1.39`.** Ignored, and logs a startup warning. In `v1.31` through `v1.38`: if set, Weaviate tries to create a new snapshot during startup when enough new commit log data has accumulated since the last snapshot. Otherwise, it loads the existing snapshot. Default: `true`. See [snapshot configuration before `v1.39`](/weaviate/concepts/storage.md#pre-v1-39-configuration).
Added in `v1.31` | `boolean` | `false` | | `PERSISTENCE_HNSW_MAX_LOG_SIZE` | Maximum size of the HNSW [write-ahead-log](/weaviate/concepts/storage.md#hnsw-vector-index-storage). Increase this to improve log compaction efficiency, or decrease to reduce memory requirements. Default: 500MiB | `string` | `4GiB` (IEC units), `4GB` (SI units), `4000000000` (bytes) | | `PERSISTENCE_LSM_ACCESS_STRATEGY` | Function used to access disk data in virtual memory. Default: `mmap` | `string` | `mmap` or `pread` | | `PERSISTENCE_LSM_MAX_SEGMENT_SIZE` | Maximum size of a segment in the [LSM store](/weaviate/concepts/storage.md#object-and-inverted-index-store). Set this to limit disk usage spikes during compaction to ~2x the segment size. Default: no limit | `string` | `4GiB` (IEC units), `4GB` (SI units), `4000000000` (bytes) | diff --git a/docs/deploy/configuration/env-vars/runtime-config.md b/docs/deploy/configuration/env-vars/runtime-config.md index 209c1b197..722cafaea 100644 --- a/docs/deploy/configuration/env-vars/runtime-config.md +++ b/docs/deploy/configuration/env-vars/runtime-config.md @@ -11,7 +11,7 @@ import RuntimeConfig from '/_includes/feature-notes/runtime-config.mdx'; Weaviate supports runtime configuration management, allowing some configurations to be changed without any further restarts. -Each runtime configuration corresponds to an existing environment variable. When a runtime configuration is updated, it overrides the value set by the corresponding environment variable. +Most runtime configurations correspond to an existing environment variable. When a runtime configuration is updated, it overrides the value set by the corresponding environment variable. ## How to set up runtime configuration @@ -70,6 +70,7 @@ The following overrides are currently supported: | `export_default_path` | `EXPORT_DEFAULT_PATH` | | `export_enabled` | `EXPORT_ENABLED` | | `export_parallelism` | `EXPORT_PARALLELISM` | +| `grpc_web_enabled` | _(not applicable)_ | | `inverted_sorter_disabled` | `INVERTED_SORTER_DISABLED` | | `maximum_allowed_collections_count` | `MAXIMUM_ALLOWED_COLLECTIONS_COUNT` | | `objects_ttl_batch_size` | `OBJECTS_TTL_BATCH_SIZE` | diff --git a/docs/deploy/configuration/monitoring.md b/docs/deploy/configuration/monitoring.md index fcec0e0db..4cecd3f0d 100644 --- a/docs/deploy/configuration/monitoring.md +++ b/docs/deploy/configuration/monitoring.md @@ -282,14 +282,6 @@ These metrics track Write-Ahead Log (WAL) recovery operations during startup. | `schema_reads_leader_seconds` | Duration of schema reads that are passed to the leader | `type` | `Summary` | | `schema_wait_for_version_seconds` | Duration of waiting for a schema version to be reached | `type` | `Summary` | -#### Schema transactions (deprecated) - -| Metric | Description | Labels | Type | -| ---------------------------- | --------------------------------------------------------------------------------------- | --------------------- | --------- | -| `schema_tx_opened_total` | Total number of opened schema transactions | `ownership` | `Counter` | -| `schema_tx_closed_total` | Total number of closed schema transactions. A close must be either successful or failed | `ownership`, `status` | `Counter` | -| `schema_tx_duration_seconds` | Mean duration of a tx by status | `ownership`, `status` | `Summary` | - #### RAFT metrics (internal) | Metric | Description | Labels | Type | diff --git a/docs/deploy/configuration/persistence.md b/docs/deploy/configuration/persistence.md index e503b049e..074904dcb 100644 --- a/docs/deploy/configuration/persistence.md +++ b/docs/deploy/configuration/persistence.md @@ -107,6 +107,7 @@ You can configure automatic deletion of objects after a specified time period us ## Related pages - [Configuration: Backups](/deploy/configuration/backups.md) +- [Concepts: Storage - HNSW snapshots](/weaviate/concepts/storage.md#hnsw-snapshots) ## Questions and feedback diff --git a/docs/deploy/configuration/replica-movement.mdx b/docs/deploy/configuration/replica-movement.mdx index 87f874e69..86899d478 100644 --- a/docs/deploy/configuration/replica-movement.mdx +++ b/docs/deploy/configuration/replica-movement.mdx @@ -146,10 +146,13 @@ The movement operation can have one of the following states: - `REGISTERED` - `HYDRATING` - `FINALIZING` +- `INTEGRATING` - `DEHYDRATING` - `READY` - `CANCELLED` +`INTEGRATING` was added in `v1.38.0`. On earlier versions the operation goes straight from `FINALIZING` to `DEHYDRATING` for a move, or to `READY` for a copy. + To learn more about the replication states, check out [Concepts: Replication architecture](/docs/weaviate/concepts/replication-architecture/consistency.md#replica-movement). ::: diff --git a/docs/weaviate/api/graphql/search-operators.md b/docs/weaviate/api/graphql/search-operators.md index 7cd5023a7..b2474aad4 100644 --- a/docs/weaviate/api/graphql/search-operators.md +++ b/docs/weaviate/api/graphql/search-operators.md @@ -435,9 +435,15 @@ To mitigate this effect, Weaviate automatically performs a search with a higher -Use `bm25SearchOperator` to set how many of the query tokens must be present within a single searched property for an object to be considered a match in the keyword (bm25) search portion of the hybrid search. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. +Use `bm25SearchOperator` to set how the query tokens must match for an object to be considered a match in the keyword (bm25) search portion of the hybrid search. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. -The available options are `And`, or `Or`. With `And`, all of the query tokens must appear together within a single searched property; tokens spread across different properties do not match. If `Or` is set, an additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. +The available options are `And`, `Or`, and `AndCross`: + +| Option | Description | +| ------ | ----------- | +| `And` | All of the query tokens must appear together within a single searched property. Tokens spread across different properties do not match. | +| `Or` | An additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. | +| `AndCross` | Every query token must be matched by at least one of the searched properties, so the tokens can be spread across different properties. All searched properties must share the same tokenization and analyzer settings, otherwise the query fails with an error. (available from `v1.38.8`) | If not set, the keyword search behaves as if `Or` was set with a `minimumOrTokensMatch` of `1`. @@ -458,7 +464,7 @@ The `bm25` operator supports the following variables: | --------- | -------- | ----------- | | `query` | yes | The keyword search query. | | `properties` | no | Array of properties (fields) to search in, defaulting to all properties in the collection. | -| `searchOperator` | no | set how many of the query tokens must be present within a single searched property for an object to be considered a match. (available from `v1.31.0`) | +| `searchOperator` | no | Set how the query tokens must match for an object to be considered a match. See [Search operator](#search-operator) for the available options. (available from `v1.31.0`) | :::info Boosting properties Specific properties can be boosted by a factor specified as a number after the caret sign, for example `properties: ["title^3", "summary"]`. @@ -537,12 +543,39 @@ import GraphQLFiltersBM25FilterExample from '/_includes/code/graphql.filters.bm2 -Use `searchOperator` to set how many of the query tokens must be present within a single searched property for an object to be considered a match. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. +Use `searchOperator` to set how the query tokens must match for an object to be considered a match. This is useful when you want to ensure that only objects with a certain number of relevant keywords are returned. + +The available options are `And`, `Or`, and `AndCross`: -The available options are `And`, or `Or`. With `And`, all of the query tokens must appear together within a single searched property; tokens spread across different properties do not match. If `Or` is set, an additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. +| Option | Description | +| ------ | ----------- | +| `And` | All of the query tokens must appear together within a single searched property. Tokens spread across different properties do not match. | +| `Or` | An additional parameter `minimumOrTokensMatch` must be specified, which defines how many of the query tokens must be present within a single searched property for the object to be considered a match. | +| `AndCross` | Every query token must be matched by at least one of the searched properties, so the tokens can be spread across different properties. All searched properties must share the same tokenization and analyzer settings, otherwise the query fails with an error. (available from `v1.38.8`) | If not set, the keyword search behaves as if `Or` was set with a `minimumOrTokensMatch` of `1`. +An `AndCross` query example: + +```graphql +{ + Get { + JeopardyQuestion( + limit: 3 + bm25: { + query: "African desert wind" + searchOperator: { + operator: AndCross + } + } + ) { + question + answer + } + } +} +``` + ## ask Enabled by the module: [Question Answering](/weaviate/modules/qna-transformers.md). diff --git a/docs/weaviate/api/grpc.md b/docs/weaviate/api/grpc.md index 57b20cd39..ccba6f0ea 100644 --- a/docs/weaviate/api/grpc.md +++ b/docs/weaviate/api/grpc.md @@ -53,6 +53,19 @@ Alternatively, you can use other tools, such as the `grpcurl` command-line tool, - `grpcurl` command-line tool ([GitHub repo](https://github.com/fullstorydev/grpcurl)) - Postman ([How to send a gRPC request with Postman](https://learning.postman.com/docs/sending-requests/grpc/grpc-request-interface/)) +## gRPC-Web + +:::info Added in `v1.38.3` +::: + +Browsers cannot speak plain gRPC. To reach the gRPC API from a browser, Weaviate also serves a gRPC-Web interface over ordinary HTTP. It is served under the `/v1/grpc-web/` path prefix on the same port as the REST API (default `8080`), not on the gRPC port, so there is no second port to expose. + +The gRPC-Web interface is enabled by default. **[Runtime configuration](/deploy/configuration/env-vars/runtime-config.md) override:** set `grpc_web_enabled` to `false`. Note the snake_case. This takes effect without a restart. + +This setting has no environment variable equivalent. When the interface is disabled, requests to `/v1/grpc-web/` fall through to the REST handler, so other REST endpoints keep working as usual. + +The Weaviate client libraries connect over plain gRPC, so they do not use the gRPC-Web interface yet. + ## Questions and feedback import DocsFeedback from "/\_includes/docs-feedback.mdx"; diff --git a/docs/weaviate/client-libraries/python/notes-best-practices.mdx b/docs/weaviate/client-libraries/python/notes-best-practices.mdx index e70a2a025..d54190bd3 100644 --- a/docs/weaviate/client-libraries/python/notes-best-practices.mdx +++ b/docs/weaviate/client-libraries/python/notes-best-practices.mdx @@ -275,7 +275,7 @@ Note that these lists are reset when a batching process is initialized. So make language="py" /> -`collection.data.ingest()` does not use a batching context, so it reports failures through its return value instead. Check `result.errors`, a dictionary that holds one entry per failed object, keyed by the position of the object in the input. The [one-shot ingest](#one-shot-ingest) example above shows this pattern. +`collection.data.ingest()` does not use a batching context, so it reports failures through its return value instead. Check `result.has_errors` for a quick summary flag that tells you whether anything failed. For the detail, check `result.errors`, a dictionary that holds one entry per failed object, keyed by the position of the object in the input. The [one-shot ingest](#one-shot-ingest) example above shows this pattern. ### Batch vectorization diff --git a/docs/weaviate/concepts/replication-architecture/consistency.md b/docs/weaviate/concepts/replication-architecture/consistency.md index 7f28f9be0..122e21103 100644 --- a/docs/weaviate/concepts/replication-architecture/consistency.md +++ b/docs/weaviate/concepts/replication-architecture/consistency.md @@ -355,6 +355,8 @@ Each replica movement operation progresses through a workflow designed to mainta - **FINALIZING**: The bulk data transfer is complete, and the new replica is catching up on any writes that occurred during the transfer. This ensures the replica is fully synchronized with the latest data. You can use the [`REPLICA_MOVEMENT_MINIMUM_ASYNC_WAIT` environment variable](/docs/deploy/configuration/env-vars/index.md#REPLICA_MOVEMENT_MINIMUM_ASYNC_WAIT) to adjust the wait time which ensures that any in progress writes have been completed and replicated to the target node. +- **INTEGRATING** (added in `v1.38.0`): The new replica has joined the shard's replica set and is being brought into the write path on every node. The operation waits until all nodes agree that the new replica is a write target, so that no node can acknowledge a write that skips it. Each node reports that it reached this state only after its own in-flight writes to the shard have drained, which prevents a write that was already accepted from being lost. Once the cluster agrees, the last writes recorded on the source during the transition are applied to the new replica, and the source stops recording further changes for this operation. For copy operations, the next state is **READY**. For move operations, the next state is **DEHYDRATING**. + - **DEHYDRATING**: For move operations, after the new replica is ready, the original replica on the source node is being removed. - **READY**: The operation has completed successfully. The new replica is fully synchronized and ready to serve traffic. For move operations, the source replica has been removed. diff --git a/docs/weaviate/concepts/search/hybrid-search.md b/docs/weaviate/concepts/search/hybrid-search.md index 181fe96b5..eeed33e5e 100644 --- a/docs/weaviate/concepts/search/hybrid-search.md +++ b/docs/weaviate/concepts/search/hybrid-search.md @@ -191,7 +191,7 @@ This is because BM25 scores are not normalized or bounded like vector distances, ## Keyword (BM25) search parameters -Hybrid search in Weaviate supports all the parameters available for keyword (BM25) search. This includes, for example, the ability to set the tokenization method, stopwords, BM25 parameters (k1, b), search operators (`and` or `or`), specific properties to search and/or to boost particular properties. +Hybrid search in Weaviate supports all the parameters available for keyword (BM25) search. This includes, for example, the ability to set the tokenization method, stopwords, BM25 parameters (k1, b), [search operators](./keyword-search.md#keyword-search-operators) (`and`, `or`, or `and_cross`), specific properties to search and/or to boost particular properties. For more information on these parameters, see the [keyword search page](./keyword-search.md). diff --git a/docs/weaviate/concepts/search/keyword-search.md b/docs/weaviate/concepts/search/keyword-search.md index 86c3a0ef9..53bebf4b6 100644 --- a/docs/weaviate/concepts/search/keyword-search.md +++ b/docs/weaviate/concepts/search/keyword-search.md @@ -136,13 +136,14 @@ import SearchOperators from '/_includes/feature-notes/search-operators.mdx'; -Search operators define the minimum number of query [tokens](../../search/bm25.md#set-tokenization) that must be present within a single searched property for an object to be returned. +Search operators define how many of the query [tokens](../../search/bm25.md#set-tokenization) must match, and whether they must all match within a single searched property. Conceptually, it works as though a filter is applied to the results of the BM25 score calculation. The available operators are: - `and`: All tokens must be present within a single searched property - `or`: At least one token must be present within a single searched property, with the minimum number of tokens being configurable (`minimumOrTokensMatch`) +- `and_cross`: Every token must be matched by at least one of the searched properties, so the tokens can be spread across different properties. All searched properties must share the same tokenization and analyzer settings, otherwise the query fails with an error. (available from `v1.38.8`) -As an example, a BM25 query of `computer networking guide` with the `and` operator would only return objects where all of the tokens `computer`, `networking`, and `guide` appear together within a single searched property. If the tokens are spread across different properties (for example, `computer` in `title` and `networking` in `description`), the object does not match under `and`. In contrast, the same query with the `or` operator would return objects where at least one of those tokens appears in a searched property. If the `or` operator is used with a `minimumOrTokensMatch` of `2`, then at least two of the tokens must be present within a single searched property. +As an example, a BM25 query of `computer networking guide` with the `and` operator would only return objects where all of the tokens `computer`, `networking`, and `guide` appear together within a single searched property. If the tokens are spread across different properties (for example, `computer` in `title` and `networking guide` in `description`), the object does not match under `and`. That restriction is specific to `and`; the same object does match under `and_cross`, which requires each token to appear in at least one of the searched properties rather than all of them in the same one. In contrast, the same query with the `or` operator would return objects where at least one of those tokens appears in a searched property. If the `or` operator is used with a `minimumOrTokensMatch` of `2`, then at least two of the tokens must be present within a single searched property. If not specified, the default operator is `or`, with a `minimumOrTokensMatch` of `1`. This means that at least one token must be present in a searched property for the object to be returned. diff --git a/docs/weaviate/concepts/storage.md b/docs/weaviate/concepts/storage.md index 7c7607416..a673277e7 100644 --- a/docs/weaviate/concepts/storage.md +++ b/docs/weaviate/concepts/storage.md @@ -1,7 +1,7 @@ --- title: Storage sidebar_position: 18 -description: "Persistent, fault-tolerant storage architecture for objects, vectors, and inverted index management." +description: "Persistent, fault-tolerant storage architecture for objects, vectors, and inverted index management, including HNSW snapshots and commit log compaction." image: og/docs/concepts.jpg # tags: ['architecture', 'storage'] --- @@ -13,7 +13,7 @@ The components mentioned on this page aid Weaviate in creating some of its uniqu * Each write operation is immediately persisted and also tolerant to application and system crashes. * On a vector search query, Weaviate returns the entire object (in other databases sometimes called a "document"), not just a reference, such as an ID. * When combining structured search with vector search, filters are applied prior to performing the vector search. This means that you will always receive the specified number of elements as opposed to post-filtering when the final result count is unpredictable. -* Objects and their vectors can be updated or deleted at will; even while reading from the database. +* Objects and their vectors can be updated or deleted at will, even while reading from the database. ## Logical Storage Units: Indexes, Shards, Stores @@ -35,7 +35,7 @@ Weaviate periodically merges smaller, older segments to make larger segments. Si Considerations -Object storage and inverted index storage implement the LSM algorithm; they use segmentation. The vector index uses a different storage algorithm. The vector index does not use segmentation. +Object storage and inverted index storage implement the LSM algorithm, they use segmentation. The vector index uses a different storage algorithm. The vector index does not use segmentation. Weaviate versions before `v1.5.0` use a B+Tree storage mechanism. The LSM method is faster, it works in constant time, and it improves write performance. @@ -93,7 +93,7 @@ Prior to v1.36.6, lazy shard loading was enabled by default for all collections. Both the LSM stores used for object and inverted storage, as well as the HNSW vector index store make use of memory at some point of the ingestion journey. To prevent data loss on a crash, each operation is additionally written into a **[Write-Ahead-Log (WAL)](https://martinfowler.com/articles/patterns-of-distributed-systems/wal.html)** (also known as a *commit log*). WALs are append-only files that are very efficient to write to and that are rarely a bottleneck for ingestion. -By the time Weaviate has responded with a successful status to your ingestion request, a WAL entry will have been created. If a WAL entry could not be created - for example because the disks are full - Weaviate will respond with an error to the insert or update request. +By the time Weaviate has responded with a successful status to your ingestion request, an LSM store WAL entry will have been created. If a WAL entry could not be created - for example because the disks are full - Weaviate will respond with an error to the insert or update request. The HNSW vector index keeps its own commit log, described [below](#hnsw-snapshots). It is written on the same request path, and the two differ in when they are synced to disk. The LSM stores will try to flush a segment on an orderly shutdown. Only if the operation is successful, will the WAL be marked as "complete". This means that if an unexpected crash happens and Weaviate encounters an "incomplete" WAL, it will recover from it. As part of the recovery process, Weaviate will flush a new segment based on the WAL and mark it as complete. As a result, future restarts will no longer have to recover from this WAL. @@ -101,7 +101,7 @@ For the HNSW vector index, the Write-Ahead-Log (WAL) is a critical component for The entire HNSW index state can be reconstructed by replaying these WAL entries. -For very large indexes of tens or hundreds of millions of objects, this can be time-consuming. If you have a large index and you want to speed up the startup time, you can use the **[HNSW snapshots](../configuration/hnsw-snapshots.md)** feature. +For very large indexes of tens or hundreds of millions of objects, this can be time-consuming. To avoid replaying the entire commit log on every restart, Weaviate writes **[HNSW snapshots](#hnsw-snapshots)**. ### HNSW snapshots @@ -109,31 +109,52 @@ import HnswSnapshots from '/_includes/feature-notes/hnsw-snapshots.mdx'; -For very large HNSW vector indexes, HNSW snapshots can significantly reduce the startup time. +A snapshot represents a point-in-time state of the HNSW index. When Weaviate starts, it loads the most recent snapshot and replays only the commit log entries written after it. This significantly reduces startup time, because the number of entries that have to be replayed no longer grows with the age of the index. -A snapshot represents a point-in-time state of the HNSW index. When Weaviate starts, if a valid snapshot exists, it will be loaded into memory first. This significantly reduces startup time, as the number of WAL entries that need to be processed, as only the changes made after the snapshot was taken need to be replayed from the WAL. +The commit log records every change to the index as it happens. Entries are written to the log as batches are processed, and a log file is synced to disk when it is rotated. Even with a fresh snapshot, Weaviate typically still has to load at least one subsequent commit log file. -If a snapshot cannot be loaded for any reason, it is safely removed, and Weaviate falls back to the traditional method of loading the full commit log from the beginning, ensuring resilience. +Starting in `v1.39`, snapshots are part of how the vector index is stored rather than an optional speedup. A background process called the commit log compactor owns the on-disk lifecycle of the index: it compacts newly flushed commit logs, merges them together, and writes a new snapshot when doing so is worthwhile. Snapshots and commit logs live in the same directory, and a snapshot replaces the commit logs it covers rather than duplicating them, so the commit logs left on disk hold only the delta since the last snapshot. This keeps the disk footprint proportional to the size of the index. Snapshots are also written as a stream. Weaviate still loads the snapshot it supersedes into memory, but the commit log delta and the new snapshot itself are streamed rather than also held there, as they were before `v1.39`. -Snapshots can be created at startup and periodically based on time passed or changes in the commit log. +Upgrading to `v1.39` reduces the disk space the vector index uses, in some cases substantially. Earlier versions keep the full commit log alongside the snapshot, and a snapshot is a more compact representation of the same index than the commit logs it replaces, because compaction keeps only the final state of each vector's connections instead of every change made to them. -Weaviate will try to create a new snapshot during startup if there are changes in the commit log since the last snapshot. If there are no changes, then the existing snapshot will be loaded. +A few caveats apply. The saving appears once the compactor has run its first cycles on each loaded shard rather than at the moment you upgrade, and inactive tenants do not shrink until they are next activated. Plan headroom for the peak rather than the steady state: while a snapshot is being written, the directory transiently holds the previous snapshot, the files being merged, and the new snapshot as it is assembled, so disk usage during snapshot creation is meaningfully above the size the index settles at. -A snapshot will also be created if the corresponding conditions are met, meaning the specified time interval has passed and a sufficient number of new commits exist. This is handled by the same background process that manages commit log combining and condensing, ensuring stability as the commit logs used for snapshot creation are not mutable during this process. +Weaviate protects this on-disk state in several ways. Snapshots and compacted commit logs are written to a temporary path and atomically renamed into place, so an interrupted write can never be mistaken for a complete file, and orphaned temporary files are cleaned up on the next startup. When a new snapshot is written, the snapshot it supersedes and the commit logs it covers are removed only after the new one is durably on disk. -Each new HNSW snapshot is based on the previous snapshot and newer (delta) commit logs. +Commit logs are self-healing. If a crash leaves the last entry of a log incomplete, the file is truncated back to its last valid entry. The entries written before the tear are retained and the file becomes valid again for later compaction, so only the incomplete tail is lost. -It's important to note that even with a fresh snapshot, the server typically still has to load at least one subsequent commit log file. +Snapshots are handled differently. A snapshot is stored in a checksummed block format and every block is verified when it is read, but unlike a commit log, a snapshot is not truncated or repaired. -The WAL is still used to persist every change immediately, guaranteeing that any acknowledged write is durable. Over time, the append-only WAL will contain redundant information for operations occurring after the last snapshot. A background process continuously compacts these newer WAL files, removing redundant information. This, combined with snapshotting, keeps the disk footprint manageable and startup times fast. +In the rare case that the current snapshot cannot be read, restore the affected data from a [backup](/deploy/configuration/backups.md), which includes the snapshot. Weaviate does not load a partial index, and because the commit logs the snapshot covers have already been removed, nothing remains on the node to replay in its place. -See **[the HNSW snapshots configuration](../configuration/hnsw-snapshots.md)** for more details on how to configure this feature. +That failure is scoped to the shard that owns the snapshot: the shard fails to load, and so does every other vector index on it. If that shard uses [dynamic lazy shard loading](#dynamic-lazy-shard-loading), the node stays up and requests to the shard return an error. If the shard is loaded eagerly, which is the default for single-tenant collections and for multi-tenant collections below the auto-detection thresholds, node startup fails instead. -Starting in `v1.36`, HNSW snapshots are enabled by default. In `v1.31` through `v1.35`, they are disabled by default. +Weaviate creates and maintains snapshots automatically, so there is nothing to enable, disable, schedule, or tune. [`PERSISTENCE_HNSW_MAX_LOG_SIZE`](/deploy/configuration/env-vars/index.md#PERSISTENCE_HNSW_MAX_LOG_SIZE) still influences the size at which commit log files are rotated, and therefore how often there is new material to compact, but it does not configure snapshots. + +The environment variables that configured snapshots before `v1.39` are deprecated. That version and later still recognize `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` and the `PERSISTENCE_HNSW_SNAPSHOT_*` variables, so an existing deployment starts without a configuration error, but their values are ignored. For each of these variables that is set, Weaviate logs a warning at startup stating that the variable has no effect and will be removed in a future version. If these options are set through a configuration file rather than as environment variables, they are ignored in the same way, but no startup warning is logged. Remove the variables from your deployment configuration to clear the warnings. + +#### Snapshot configuration before `v1.39` {#pre-v1-39-configuration} + +In `v1.31` through `v1.38`, snapshots are an optional feature layered on top of the commit log rather than part of it, and the `PERSISTENCE_HNSW_SNAPSHOT_*` environment variables control when Weaviate creates them. Snapshots are enabled by default starting in `v1.36`, and disabled by default in `v1.31` through `v1.35`. Weaviate can create one at startup and periodically thereafter, once enough new commit log data has accumulated since the last snapshot. Only commit log files that have been rotated count toward that threshold, so changes still in the active file are not considered until the next rotation. If a snapshot cannot be read in these versions, it is discarded and Weaviate replays the full commit log instead. For the variables themselves, including their defaults and deprecation status, see [`PERSISTENCE_HNSW_DISABLE_SNAPSHOTS`](/deploy/configuration/env-vars/index.md#PERSISTENCE_HNSW_DISABLE_SNAPSHOTS) and the rows that follow it. + +
+ Periodic snapshot conditions and memory requirements + +Periodic snapshot creation is governed by three variables, and **all** of the following conditions must be met before Weaviate creates a snapshot: + +- `PERSISTENCE_HNSW_SNAPSHOT_INTERVAL_SECONDS` — the minimum time since the previous snapshot has elapsed (default `21600` seconds, or six hours). +- `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_NUMBER` — enough new commit log files have been created since the last snapshot (default `1`). +- `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_SIZE_PERCENTAGE` — the new commit logs are large enough, measured as a percentage of the previous snapshot's size (default `5`). This condition does not apply to the first snapshot, when there is no previous snapshot to measure against. + +Meeting these conditions makes a snapshot eligible rather than guaranteed. The background process that condenses and combines commit log files is also the one that writes the snapshot, so a snapshot can be created on a later pass than the one where the conditions are first met. + +In these versions, before creating a new snapshot, Weaviate loads the previous snapshot and the commit log difference into memory, so the node needs enough memory to accommodate both. + +
## Conclusions -This page introduced you to the storage mechanisms of Weaviate. It outlined how all writes are persisted immediately and outlined the patterns used within Weaviate to make datasets scale well. For structured data, Weaviate makes use of segmentation to keep the write times constant. For the HNSW vector index, Weaviate avoids segmentation to keep query times efficient. +This page introduced you to the storage mechanisms of Weaviate. It outlined how all writes are persisted to a log before they are acknowledged and outlined the patterns used within Weaviate to make datasets scale well. For structured data, Weaviate makes use of segmentation to keep the write times constant. For the HNSW vector index, Weaviate avoids segmentation to keep query times efficient. ## Questions and feedback diff --git a/docs/weaviate/configuration/hnsw-snapshots.md b/docs/weaviate/configuration/hnsw-snapshots.md deleted file mode 100644 index 9db38efb5..000000000 --- a/docs/weaviate/configuration/hnsw-snapshots.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: HNSW Snapshots -sidebar_position: 47 -sidebar_label: HNSW Snapshots -description: Learn about HNSW snapshots in Weaviate for faster startup times and how to manage them. ---- - -import HnswSnapshots from '/_includes/feature-notes/hnsw-snapshots.mdx'; - - - -HNSW (Hierarchical Navigable Small World) snapshots can significantly reduce startup times for instances with large vector indexes. - -HNSW snapshotting is **enabled by default** starting in `v1.36`. To disable it, set `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` to `true`. - -In versions prior to `v1.36`, HNSW snapshotting is disabled by default. Set `PERSISTENCE_HNSW_DISABLE_SNAPSHOTS` to `false` to enable it. - -:::info Concepts: HNSW snapshots -See this [concepts page](../concepts/storage.md#hnsw-snapshots) for a detailed description. -::: - -## Configuring snapshot creation - -Set the following optional environment variables to configure the snapshotting behavior. - -:::note -Before creating a new snapshot, the previous snapshot and the commit log difference need to be loaded into memory. Make sure you have enough memory to accommodate this process. -::: - -### Snapshot on startup - -Enable or disable snapshot creation on startup: - -- `PERSISTENCE_HNSW_SNAPSHOT_ON_STARTUP`: If `true`, Weaviate will try to create a new snapshot during startup if there are changes in the commit log since the last snapshot. If there are no changes, then the existing snapshot will be loaded. - - **Default:** `true` - -### Periodic snapshots - -Set the following to configure periodic snapshot creation. Note **all** of the following conditions must be met to trigger a snapshot: - -1. **A time interval has passed:** - - - `PERSISTENCE_HNSW_SNAPSHOT_INTERVAL_SECONDS`: The minimum time in seconds since the previous snapshot. - - **Default:** `21600` seconds (6 hours) - -2. **Sufficient new commit logs (by number):** - - - `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_NUMBER`: The minimum number of new commit log files created since the last snapshot. - - **Default:** `1` - -3. **Sufficient new commit logs (by size percentage):** - - `PERSISTENCE_HNSW_SNAPSHOT_MIN_DELTA_COMMITLOGS_SIZE_PERCENTAGE`: The minimum total size of new commit logs (as a percentage of the previous snapshot's size) required to trigger a new snapshot. - - **Default:** `5` (meaning 5% of the previous snapshot's size in new commit logs). For example, if the previous snapshot was 1000MB, at least 50MB of new commit log data is required. - -## Further resources - -- [Concepts: Storage - Persistence and Crash Recovery](../concepts/storage.md#persistence-and-crash-recovery) - -## Questions and feedback - -import DocsFeedback from '/\_includes/docs-feedback.mdx'; - - diff --git a/docs/weaviate/configuration/index.mdx b/docs/weaviate/configuration/index.mdx index 40bc472d4..b28eff66e 100644 --- a/docs/weaviate/configuration/index.mdx +++ b/docs/weaviate/configuration/index.mdx @@ -25,13 +25,6 @@ export const configOpsData = [ link: "/weaviate/configuration/compression", icon: "fas fa-compress-alt", }, - { - title: "HNSW Snapshots", - description: - "Configure HNSW index snapshots for faster recovery and startup.", - link: "/weaviate/configuration/hnsw-snapshots", - icon: "fas fa-camera", - }, { title: "Modules", description: diff --git a/docs/weaviate/model-providers/_includes/integration_deepseek_rag.png b/docs/weaviate/model-providers/_includes/integration_deepseek_rag.png new file mode 100644 index 000000000..81825f35c Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_deepseek_rag.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_deepseek_rag_grouped.png b/docs/weaviate/model-providers/_includes/integration_deepseek_rag_grouped.png new file mode 100644 index 000000000..7e79d2ebc Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_deepseek_rag_grouped.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_deepseek_rag_single.png b/docs/weaviate/model-providers/_includes/integration_deepseek_rag_single.png new file mode 100644 index 000000000..14d6acd8c Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_deepseek_rag_single.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_morph_embedding.png b/docs/weaviate/model-providers/_includes/integration_morph_embedding.png new file mode 100644 index 000000000..403b50546 Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_morph_embedding.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_morph_embedding_search.png b/docs/weaviate/model-providers/_includes/integration_morph_embedding_search.png new file mode 100644 index 000000000..d92cd9571 Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_morph_embedding_search.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding.png b/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding.png new file mode 100644 index 000000000..cc368b008 Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding.png differ diff --git a/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding_search.png b/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding_search.png new file mode 100644 index 000000000..756e50b84 Binary files /dev/null and b/docs/weaviate/model-providers/_includes/integration_twelvelabs_embedding_search.png differ diff --git a/docs/weaviate/model-providers/_includes/provider.connect.py b/docs/weaviate/model-providers/_includes/provider.connect.py index 022ba957d..0843511ed 100644 --- a/docs/weaviate/model-providers/_includes/provider.connect.py +++ b/docs/weaviate/model-providers/_includes/provider.connect.py @@ -33,6 +33,10 @@ # Recommended: save sensitive data as environment variables databricks_token = os.getenv("DATABRICKS_TOKEN") # END DatabricksInstantiation +# START DeepseekInstantiation +# Recommended: save sensitive data as environment variables +deepseek_key = os.getenv("DEEPSEEK_APIKEY") +# END DeepseekInstantiation # START DigitalOceanInstantiation # Recommended: save sensitive data as environment variables digitalocean_key = os.getenv("DIGITALOCEAN_APIKEY") @@ -63,6 +67,10 @@ # Recommended: save sensitive data as environment variables mistral_key = os.getenv("MISTRAL_API_KEY") # END MistralInstantiation +# START MorphInstantiation +# Recommended: save sensitive data as environment variables +morph_key = os.getenv("MORPH_APIKEY") +# END MorphInstantiation # START NVIDIAInstantiation # Recommended: save sensitive data as environment variables nvidia_key = os.getenv("NVIDIA_API_KEY") @@ -79,6 +87,10 @@ # Recommended: save sensitive data as environment variables azure_key = os.getenv("AZURE_API_KEY") # END AzureOpenAIInstantiation +# START TwelveLabsInstantiation +# Recommended: save sensitive data as environment variables +twelvelabs_key = os.getenv("TWELVELABS_APIKEY") +# END TwelveLabsInstantiation # START VoyageAIInstantiation # Recommended: save sensitive data as environment variables voyageai_key = os.getenv("VOYAGEAI_API_KEY") @@ -114,6 +126,10 @@ # START DatabricksInstantiation "X-Databricks-Token": databricks_token, # END DatabricksInstantiation +# START DeepseekInstantiation + "X-Deepseek-Api-Key": deepseek_key, + # "X-Deepseek-Baseurl": "https://api.deepseek.com", # Optional; for providing a custom base URL +# END DeepseekInstantiation # START DigitalOceanInstantiation "X-Digitalocean-Api-Key": digitalocean_key, # END DigitalOceanInstantiation @@ -138,6 +154,11 @@ # START MistralInstantiation "X-Mistral-Api-Key": mistral_key, # END MistralInstantiation +# START MorphInstantiation + # Morph requests are built by Weaviate's OpenAI-compatible client, + # so the Morph key is supplied under the OpenAI header name. + "X-Openai-Api-Key": morph_key, +# END MorphInstantiation # START NVIDIAInstantiation "X-NVIDIA-Api-Key": nvidia_key, # END NVIDIAInstantiation @@ -150,6 +171,10 @@ # START AzureOpenAIInstantiation "X-Azure-Api-Key": azure_key, # END AzureOpenAIInstantiation +# START TwelveLabsInstantiation + "X-Twelvelabs-Api-Key": twelvelabs_key, + "X-Twelvelabs-Baseurl": "https://api.twelvelabs.io/v1.3", # Optional; for providing a custom base URL +# END TwelveLabsInstantiation # START VoyageAIInstantiation "X-VoyageAI-Api-Key": voyageai_key, # END VoyageAIInstantiation diff --git a/docs/weaviate/model-providers/_includes/provider.connect.ts b/docs/weaviate/model-providers/_includes/provider.connect.ts index 149bf866a..8ab469616 100644 --- a/docs/weaviate/model-providers/_includes/provider.connect.ts +++ b/docs/weaviate/model-providers/_includes/provider.connect.ts @@ -42,6 +42,9 @@ const jinaaiApiKey = process.env.JINAAI_API_KEY || ''; // Replace with your inf // START MistralInstantiation const mistralApiKey = process.env.MISTRAL_API_KEY || ''; // Replace with your inference API key // END MistralInstantiation +// START MorphInstantiation +const morphApiKey = process.env.MORPH_APIKEY || ''; // Replace with your inference API key +// END MorphInstantiation // START NVIDIAInstantiation const nvidiaApiKey = process.env.NVIDIA_API_KEY || ''; // Replace with your inference API key // END NVIDIAInstantiation @@ -113,6 +116,11 @@ const client = await weaviate.connectToWeaviateCloud( // START MistralInstantiation 'X-Mistral-Api-Key': mistralApiKey, // END MistralInstantiation + // START MorphInstantiation + // Morph requests are built by Weaviate's OpenAI-compatible client, + // so the Morph key is supplied under the OpenAI header name. + 'X-Openai-Api-Key': morphApiKey, + // END MorphInstantiation // START NVIDIAInstantiation 'X-NVIDIA-Api-Key': nvidiaApiKey, // END NVIDIAInstantiation diff --git a/docs/weaviate/model-providers/_includes/provider.generative.py b/docs/weaviate/model-providers/_includes/provider.generative.py index 4cd4d43f2..bf5daff80 100644 --- a/docs/weaviate/model-providers/_includes/provider.generative.py +++ b/docs/weaviate/model-providers/_includes/provider.generative.py @@ -858,6 +858,94 @@ def import_data(): # clean up client.collections.delete("DemoCollection") +# --------------------------------------------------------------------------- +# DeepSeek generative integration (generative-deepseek). +# +# `Configure.Generative.deepseek()` and `GenerativeConfig.deepseek()` (merged +# upstream in PR #2084, commit afc0e0eb) ship in the client release pinned by +# pyproject.toml, so the guard that kept these blocks unreachable is gone. The +# calls below match the released keyword-only signatures exactly: base_url, +# model, temperature, max_tokens, frequency_penalty, presence_penalty, top_p, +# stop. +# --------------------------------------------------------------------------- +DEEPSEEK_CLIENT_AVAILABLE = True + +# NOTE: there is deliberately no "basic" no-model block for DeepSeek. The +# module's built-in default model is the retired `deepseek-chat` alias, so +# `Configure.Generative.deepseek()` with no arguments is not a usable +# example. Every documented DeepSeek block sets `model` explicitly. + +# START GenerativeDeepseekCustomModel +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + # highlight-start + generative_config=Configure.Generative.deepseek( + model="deepseek-v4-flash" + ) + # highlight-end + # Additional parameters not shown +) +# END GenerativeDeepseekCustomModel + +# clean up +client.collections.delete("DemoCollection") + +# START FullGenerativeDeepseek +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + # highlight-start + generative_config=Configure.Generative.deepseek( + model="deepseek-v4-flash", + # # These parameters are optional + # temperature=0.7, + # max_tokens=500, + # frequency_penalty=0.0, + # presence_penalty=0.0, + # top_p=1.0, + # base_url="https://api.deepseek.com", + # stop=["\n\n"], + ) + # highlight-end +) +# END FullGenerativeDeepseek + +# clean up +client.collections.delete("DemoCollection") +import_data() + +# START RuntimeModelSelectionDeepseek +from weaviate.classes.config import Configure +from weaviate.classes.generate import GenerativeConfig + +collection = client.collections.use("DemoCollection") +response = collection.generate.near_text( + query="A holiday film", + limit=2, + grouped_task="Write a tweet promoting these two movies", + # highlight-start + generative_provider=GenerativeConfig.deepseek( + # # These parameters are optional + model="deepseek-v4-pro", + # temperature=0.7, + # max_tokens=500, + # frequency_penalty=0.0, + # presence_penalty=0.0, + # top_p=1.0, + # base_url="https://api.deepseek.com", + # stop=["\n\n"], + ), + # Additional parameters not shown + # highlight-end +) +# END RuntimeModelSelectionDeepseek + +# clean up +client.collections.delete("DemoCollection") + # START BasicGenerativeNVIDIA from weaviate.classes.config import Configure diff --git a/docs/weaviate/model-providers/_includes/provider.vectorizer.py b/docs/weaviate/model-providers/_includes/provider.vectorizer.py index 1acbf121d..3d5b1d51a 100644 --- a/docs/weaviate/model-providers/_includes/provider.vectorizer.py +++ b/docs/weaviate/model-providers/_includes/provider.vectorizer.py @@ -25,7 +25,7 @@ region="us-east-1", source_properties=["title"], service="bedrock", - model="titan-embed-text-v2:0", + model="amazon.titan-embed-text-v2:0", ) ], # highlight-end @@ -62,17 +62,40 @@ # START FullVectorizerAWS from weaviate.classes.config import Configure +# For Bedrock client.collections.create( "DemoCollection", # highlight-start vector_config=[ - Configure.Vectors.text2vec_aws( + Configure.Vectors.text2vec_aws_bedrock( + name="title_vector", + region="us-east-1", + source_properties=["title"], + model="amazon.titan-embed-text-v2:0", # Required + # Further options + # dimensions=512, # Amazon models only + ) + ], + # highlight-end + # Additional parameters not shown +) + +# clean up +client.collections.delete("DemoCollection") + +# For SageMaker +client.collections.create( + "DemoCollection", + # highlight-start + vector_config=[ + Configure.Vectors.text2vec_aws_sagemaker( name="title_vector", region="us-east-1", source_properties=["title"], - service="bedrock", # `bedrock` or `sagemaker` - model="titan-embed-text-v2:0", # If using `bedrock`, this is required - # endpoint="", # If using `sagemaker`, this is required + endpoint="", # Required + # Further options + # target_model="", + # target_variant="", ) ], # highlight-end @@ -294,12 +317,13 @@ client.collections.create( "DemoCollection", # highlight-start - vector_config=Configure.Vectors.text2vec_google( + vector_config=Configure.Vectors.text2vec_google_vertex( name="title_vector", source_properties=["title"], project_id="", # Required for Vertex AI # Further options # model="", + # location="", # api_endpoint="", ), # highlight-end @@ -702,6 +726,49 @@ # clean up client.collections.delete("DemoCollection") +# START BasicVectorizerMorph +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + # highlight-start + vector_config=[ + Configure.Vectors.text2vec_morph( + name="title_vector", + source_properties=["title"], + ) + ], + # highlight-end + # Additional parameters not shown +) +# END BasicVectorizerMorph + +# clean up +client.collections.delete("DemoCollection") + +# START FullVectorizerMorph +from weaviate.classes.config import Configure + +client.collections.create( + "DemoCollection", + # highlight-start + vector_config=[ + Configure.Vectors.text2vec_morph( + name="title_vector", + source_properties=["title"], + model="morph-embedding-v3", + base_url="https://api.morphllm.com", # Base URL; an existing path is preserved + endpoint="/v1/embeddings", # Path appended to the base URL + ) + ], + # highlight-end + # Additional parameters not shown +) +# END FullVectorizerMorph + +# clean up +client.collections.delete("DemoCollection") + # START BasicVectorizerNVIDIA from weaviate.classes.config import Configure @@ -1124,6 +1191,100 @@ # clean up client.collections.delete("DemoCollection") +# START BasicMMVectorizerTwelveLabs +from weaviate.classes.config import Configure, DataType, Multi2VecField, Property + +client.collections.create( + "DemoCollection", + # highlight-start + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="poster", data_type=DataType.BLOB), + ], + vector_config=[ + Configure.Vectors.multi2vec_twelvelabs( + name="title_vector", + # Define the fields to be used for the vectorization - using image_fields, text_fields + image_fields=[ + Multi2VecField(name="poster", weight=0.9) + ], + text_fields=[ + Multi2VecField(name="title", weight=0.1) + ], + ) + ], + # highlight-end + # Additional parameters not shown +) +# END BasicMMVectorizerTwelveLabs + +# clean up +client.collections.delete("DemoCollection") + +# START MMVectorizerTwelveLabsCustomModel +from weaviate.classes.config import Configure, DataType, Multi2VecField, Property + +client.collections.create( + "DemoCollection", + # highlight-start + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="poster", data_type=DataType.BLOB), + ], + vector_config=[ + Configure.Vectors.multi2vec_twelvelabs( + name="title_vector", + model="marengo3.0", + # Define the fields to be used for the vectorization - using image_fields, text_fields + image_fields=[ + Multi2VecField(name="poster", weight=0.9) + ], + text_fields=[ + Multi2VecField(name="title", weight=0.1) + ], + ) + ], + # highlight-end + # Additional parameters not shown +) +# END MMVectorizerTwelveLabsCustomModel + +# clean up +client.collections.delete("DemoCollection") + +# START FullMMVectorizerTwelveLabs +from weaviate.classes.config import Configure, DataType, Multi2VecField, Property + +client.collections.create( + "DemoCollection", + # highlight-start + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="poster", data_type=DataType.BLOB), + ], + vector_config=[ + Configure.Vectors.multi2vec_twelvelabs( + name="title_vector", + # Define the fields to be used for the vectorization - using image_fields, text_fields + image_fields=[ + Multi2VecField(name="poster", weight=0.9) + ], + text_fields=[ + Multi2VecField(name="title", weight=0.1) + ], + # Further options + # model="marengo3.0", + # base_url="https://api.twelvelabs.io/v1.3", + ) + ], + # highlight-end + # Additional parameters not shown +) +# END FullMMVectorizerTwelveLabs + +# clean up +client.collections.delete("DemoCollection") + # START BasicVectorizerVoyageAI from weaviate.classes.config import Configure diff --git a/docs/weaviate/model-providers/_includes/provider.vectorizer.ts b/docs/weaviate/model-providers/_includes/provider.vectorizer.ts index 11f5bef7a..72b75ad0b 100644 --- a/docs/weaviate/model-providers/_includes/provider.vectorizer.ts +++ b/docs/weaviate/model-providers/_includes/provider.vectorizer.ts @@ -851,6 +851,56 @@ await client.collections.create({ // Clean up await client.collections.delete('DemoCollection'); +// START BasicVectorizerMorph +await client.collections.create({ + name: 'DemoCollection', + properties: [ + { + name: 'title', + dataType: 'text' as const, + }, + ], + // highlight-start + vectorizers: [ + weaviate.configure.vectors.text2VecMorph({ + name: 'title_vector', + sourceProperties: ['title'], + }), + ], + // highlight-end + // Additional parameters not shown +}); +// END BasicVectorizerMorph + +// Clean up +await client.collections.delete('DemoCollection'); + +// START FullVectorizerMorph +await client.collections.create({ + name: 'DemoCollection', + properties: [ + { + name: 'title', + dataType: 'text' as const, + }, + ], + // highlight-start + vectorizers: [ + weaviate.configure.vectors.text2VecMorph({ + name: 'title_vector', + sourceProperties: ['title'], + model: 'morph-embedding-v3', + baseURL: 'https://api.morphllm.com', // Base URL; an existing path is preserved + }), + ], + // highlight-end + // Additional parameters not shown +}); +// END FullVectorizerMorph + +// Clean up +await client.collections.delete('DemoCollection'); + // START BasicVectorizerNVIDIA await client.collections.create({ name: 'DemoCollection', diff --git a/docs/weaviate/model-providers/aws/embeddings.md b/docs/weaviate/model-providers/aws/embeddings.md index 3fafe5b4d..49fec0529 100644 --- a/docs/weaviate/model-providers/aws/embeddings.md +++ b/docs/weaviate/model-providers/aws/embeddings.md @@ -185,12 +185,34 @@ import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; ### Vectorizer parameters -The following examples show how to configure AWS-specific options. +**Common parameters:** +- `service` (Optional): The AWS service to use, either `bedrock` or `sagemaker`. Defaults to `bedrock`. +- `region` (Required): The AWS region to send requests to, e.g. `us-east-1`. -The AWS region setting is required for all AWS integrations. +**Bedrock parameters:** +- `model` (Required): The full Bedrock model identifier, e.g. `amazon.titan-embed-text-v2:0`. +- `dimensions` (Optional): The size of the embedding to request from the model, e.g. `512`. -- Bedrock users must set `service` to `bedrock` and provide the `model` name. -- SageMaker users must set `service` to `sagemaker` and provide the `endpoint` address. +**SageMaker parameters:** +- `endpoint` (Required): The name of the SageMaker endpoint to invoke, e.g. `tei-xxx`. +- `targetModel` (Optional): The model to target on a multi-model endpoint. +- `targetVariant` (Optional): The production variant to target on the endpoint. + +#### `service` + +Weaviate falls back to `bedrock` when `service` is not set, so a SageMaker configuration has to set it to `sagemaker`. Any value other than `bedrock` or `sagemaker` is rejected when the collection is created. + +Some clients offer service-specific constructors, such as `text2vec_aws_bedrock` and `text2vec_aws_sagemaker`, which select the service for you. Examples built on those constructors do not pass `service` at all. Every other example on this page sets it explicitly, as does any configuration written directly against the collection definition. + +#### `dimensions` + +`dimensions` was added in `v1.36.19`, and is also available from `v1.37.10`, `v1.38.2`, and `v1.39.0` onward. + +Weaviate only forwards `dimensions` to Amazon models on Bedrock, such as the Titan and Nova embedding families. Cohere models on Bedrock, and SageMaker endpoints, accept the setting in the collection configuration but ignore it when embeddings are generated, so their vectors keep the model's default size. Check the model's documentation for the sizes it supports. + +#### Example configuration + +The following examples show how to configure AWS-specific options for each service. diff --git a/docs/weaviate/model-providers/deepseek/_category_.json b/docs/weaviate/model-providers/deepseek/_category_.json new file mode 100644 index 000000000..ca510c8e0 --- /dev/null +++ b/docs/weaviate/model-providers/deepseek/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "DeepSeek", + "position": 225.5 +} diff --git a/docs/weaviate/model-providers/deepseek/generative.md b/docs/weaviate/model-providers/deepseek/generative.md new file mode 100644 index 000000000..2dcb88d0d --- /dev/null +++ b/docs/weaviate/model-providers/deepseek/generative.md @@ -0,0 +1,204 @@ +--- +title: Generative AI +description: "Weaviate's integration with DeepSeek's API allows you to access their generative models' capabilities directly from Weaviate." +sidebar_position: 50 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'deepseek', 'generative', 'rag'] +--- + +# DeepSeek Generative AI with Weaviate + +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyConnect from '!!raw-loader!../_includes/provider.connect.py'; +import PyCode from '!!raw-loader!../_includes/provider.generative.py'; + +Weaviate's integration with DeepSeek's API allows you to access their generative models' capabilities directly from Weaviate. + +[Configure a Weaviate collection](#configure-collection) to use a generative AI model with DeepSeek. Weaviate will perform retrieval augmented generation (RAG) using the specified model and your DeepSeek API key. + +More specifically, Weaviate will perform a search, retrieve the most relevant objects, and then pass them to the DeepSeek generative model to generate outputs. + +![RAG integration illustration](../_includes/integration_deepseek_rag.png) + +:::info Code examples are Python-only for now +Examples for the other client languages will follow. +::: + +## Requirements + +### Weaviate configuration + +Your Weaviate instance must be configured with the DeepSeek generative AI integration (`generative-deepseek`) module. + +:::info Added in `v1.36.19`, `v1.37.10`, and `v1.38.2` +The `generative-deepseek` module is available from `v1.36.19` on the `v1.36` line, `v1.37.10` on the `v1.37` line, and `v1.38.2` on the `v1.38` line. Earlier patch releases on these lines do not include it. +::: + +
+ For Weaviate Cloud (WCD) users + +This integration is enabled by default on Weaviate Cloud (WCD) instances. + +
+ +
+ For self-hosted users + +- Check the [cluster metadata](/deploy/configuration/status.md#cluster-metadata) to verify if the module is enabled. +- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate. +- To enable the module, include it in the `ENABLE_MODULES` environment variable available to Weaviate, e.g. `ENABLE_MODULES="generative-deepseek"` (add it to your existing comma-separated list if other modules are enabled). + +
+ +### API credentials + +You must provide a valid DeepSeek API key to Weaviate for this integration. Go to [DeepSeek](https://platform.deepseek.com/) to sign up and obtain an API key. + +Provide the API key to Weaviate using one of the following methods: + +- Set the `DEEPSEEK_APIKEY` environment variable that is available to Weaviate. +- Provide the API key at runtime, as shown in the examples below. + + + +## Configure collection + +import MutableGenerativeConfig from '/_includes/mutable-generative-config.md'; + + + +Always set `model` explicitly. The module's built-in default is a retired model alias, so a collection configured without a `model` points at a model that DeepSeek no longer serves. See [Available models](#available-models) for the current model names. + +[Configure a Weaviate index](../../manage-collections/generative-reranker-models.mdx#specify-a-generative-model-integration) as follows to use a DeepSeek generative model: + + + +### Select a model + +Specify any current DeepSeek model name. See [Available models](#available-models) for the current names, and [Generative parameters](#generative-parameters) for the other settings you can configure alongside it. + +You can also [override the model at query time](#select-a-model-at-runtime). + +### Generative parameters + +Configure the following generative parameters to customize the model behavior. + + + +Weaviate checks `maxTokens` against a built-in ceiling only for the retired `deepseek-chat` and `deepseek-reasoner` aliases. For any current model, a `maxTokens` above the model's limit is accepted when you create the collection and fails later, as an error from DeepSeek at query time. + +For further details on model parameters, see the [DeepSeek API documentation](https://api-docs.deepseek.com/). + +## Select a model at runtime + +Aside from setting the default model provider when creating the collection, you can also override it at query time. + + + +## Header parameters + +You can provide the API key as well as some optional parameters at runtime through additional headers in the request. The following headers are available: + +- `X-Deepseek-Api-Key`: The DeepSeek API key. +- `X-Deepseek-Baseurl`: The base URL to use (e.g. a proxy) instead of the default DeepSeek URL. + +`X-Deepseek-Api-Key` takes precedence over the `DEEPSEEK_APIKEY` environment variable. The API key is never part of the collection configuration, so if neither the header nor the environment variable is set, the request fails with `api key: no api key found`. + +`X-Deepseek-Baseurl` takes precedence over a `baseURL` set at query time, which in turn takes precedence over the `baseURL` in the collection configuration. If none of them are set, Weaviate uses `https://api.deepseek.com`. Provide an API root rather than a full endpoint path, because Weaviate appends `/chat/completions` to it. + +Provide the headers as shown in the [API credentials examples](#api-credentials) above. + +## Retrieval augmented generation + +After configuring the generative AI integration, perform RAG operations, either with the [single prompt](#single-prompt) or [grouped task](#grouped-task) method. + +### Single prompt + +![Single prompt RAG integration generates individual outputs per search result](../_includes/integration_deepseek_rag_single.png) + +To generate text for each object in the search results, use the single prompt method. + +The example below generates outputs for each of the `n` search results, where `n` is specified by the `limit` parameter. + +When creating a single prompt query, use braces `{}` to interpolate the object properties you want Weaviate to pass on to the language model. For example, to pass on the object's `title` property, include `{title}` in the query. + + + +### Grouped task + +![Grouped task RAG integration generates one output for the set of search results](../_includes/integration_deepseek_rag_grouped.png) + +To generate one text for the entire set of search results, use the grouped task method. + +In other words, when you have `n` search results, the generative model generates one output for the entire group. + + + +## References + +### Available models + +Weaviate forwards the configured model name to DeepSeek as-is. There is no allowlist on the Weaviate side, so any current DeepSeek model name is accepted. + +The current model names are `deepseek-v4-flash` and `deepseek-v4-pro`. For the full list of models and pricing, see the [DeepSeek pricing page](https://api-docs.deepseek.com/quick_start/pricing) and the [DeepSeek API documentation](https://api-docs.deepseek.com/). + +:::caution The module default is a retired model +The `deepseek-chat` and `deepseek-reasoner` aliases are retired and DeepSeek no longer serves them. They previously pointed at `deepseek-v4-flash` in its non-thinking and thinking modes respectively. + +The built-in default model of the `generative-deepseek` module is still `deepseek-chat`, so a collection created without a `model` points at a retired alias. Set `model` on every collection you create. +::: + +### Reasoning models + +The `generative-deepseek` module returns only the model's message content. If you use a reasoning model, its separate reasoning (chain-of-thought) output is not surfaced through the integration. + +Reasoning models can take much longer to respond than non-reasoning models. Weaviate applies the [`MODULES_CLIENT_TIMEOUT`](/deploy/configuration/env-vars/index.md#MODULES_CLIENT_TIMEOUT) environment variable to the whole request, including reading the response, and it defaults to 50 seconds. If reasoning queries time out, raise this value on your Weaviate instance. + +## Further resources + +### Code examples + +Once the integration is configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples: + +- The [How-to: Manage collections](../../manage-collections/index.mdx) and [How-to: Manage objects](../../manage-objects/index.mdx) guides show how to perform data operations (i.e. create, read, update, delete collections and objects within them). +- The [How-to: Query & Search](../../search/index.mdx) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation. + +### References + +- [DeepSeek API documentation](https://api-docs.deepseek.com/) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/model-providers/deepseek/index.md b/docs/weaviate/model-providers/deepseek/index.md new file mode 100644 index 000000000..ae10085f6 --- /dev/null +++ b/docs/weaviate/model-providers/deepseek/index.md @@ -0,0 +1,45 @@ +--- +title: DeepSeek + Weaviate +description: "DeepSeek offers a range of models for natural language processing and generation. Weaviate seamlessly integrates with the DeepSeek API, allowing users to leverage DeepSeek's generative models directly from the Weaviate Database." +sidebar_position: 10 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'deepseek'] +--- + + + +DeepSeek offers a range of models for natural language processing and generation. Weaviate seamlessly integrates with the DeepSeek API, allowing users to leverage DeepSeek's generative models directly from the Weaviate Database. + +This integration empowers developers to build sophisticated AI-driven applications with ease. + +## Integrations with DeepSeek + +### Generative AI models for RAG + +![Single prompt RAG integration generates individual outputs per search result](../_includes/integration_deepseek_rag_single.png) + +DeepSeek's generative AI models can generate human-like text based on given prompts and contexts. + +[Weaviate's generative AI integration](./generative.md) enables users to perform retrieval augmented generation (RAG) directly from the Weaviate Database. This combines Weaviate's efficient storage and fast retrieval capabilities with DeepSeek's generative AI models to generate personalized and context-aware responses. + +[DeepSeek generative AI integration page](./generative.md) + +## Summary + +This integration enables developers to leverage DeepSeek's generative models directly within Weaviate. + +In turn, it simplifies the process of building AI-driven applications to speed up your development process, so that you can focus on creating innovative solutions. + +## Get started + +You must provide a valid DeepSeek API key to Weaviate for this integration. Go to [DeepSeek](https://platform.deepseek.com/) to sign up and obtain an API key. + +Then, go to the relevant integration page to learn how to configure Weaviate with the DeepSeek models and start using them in your applications. + +- [Generative AI](./generative.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/model-providers/index.md b/docs/weaviate/model-providers/index.md index 20c49da2b..dcd5c55e3 100644 --- a/docs/weaviate/model-providers/index.md +++ b/docs/weaviate/model-providers/index.md @@ -24,16 +24,19 @@ This enables an enhanced developed experience, such as the ability to: | [Cohere](./cohere/index.md) | [Text](./cohere/embeddings.md), [Multimodal](./cohere/embeddings-multimodal.md) | [Text](./cohere/generative.md) | [Reranker](./cohere/reranker.md) | | [Contextual AI](./contextualai/index.md) | - | [Text](./contextualai/generative.md) | [Reranker](./contextualai/reranker.md) | | [Databricks](./databricks/index.md) | [Text](./databricks/embeddings.md) | [Text](./databricks/generative.md) | - | +| [DeepSeek](./deepseek/index.md) | - | [Text](./deepseek/generative.md) | - | | [DigitalOcean](./digitalocean/index.md) | [Text](./digitalocean/embeddings.md) | - | - | | [FriendliAI](./friendliai/index.md) | - | [Text](./friendliai/generative.md) | - | | [Google](./google/index.md) | [Text](./google/embeddings.md), [Multimodal](./google/embeddings-multimodal.md) | [Text](./google/generative.md) | - | | [Hugging Face](./huggingface/index.md) | [Text](./huggingface/embeddings.md) | - | - | | [Jina AI](./jinaai/index.md) | [Text](./jinaai/embeddings.md), [Multimodal](./jinaai/embeddings-multimodal.md) | - | [Reranker](./jinaai/reranker.md) | | [Mistral](./mistral/index.md) | [Text](./mistral/embeddings.md) | [Text](./mistral/generative.md) | - | +| [Morph](./morph/index.md) | [Text](./morph/embeddings.md) | - | - | | [NVIDIA](./nvidia/index.md) | [Text](./nvidia/embeddings.md), [Multimodal](./nvidia/embeddings-multimodal.md) | [Text](./nvidia/generative.md) | [Reranker](./nvidia/reranker.md) | | [OctoAI (Deprecated)](./octoai/index.md) | [Text](./octoai/embeddings.md) | [Text](./octoai/generative.md) | - | | [OpenAI](./openai/index.md) | [Text](./openai/embeddings.md) | [Text](./openai/generative.md) | - | | [Azure OpenAI](./openai-azure/index.md) | [Text](./openai-azure/embeddings.md) | [Text](./openai-azure/generative.md) | - | +| [TwelveLabs](./twelvelabs/index.md) | [Multimodal](./twelvelabs/embeddings-multimodal.md) | - | - | | [Voyage AI](./voyageai/index.md) | [Text](./voyageai/embeddings.md), [Multimodal](./voyageai/embeddings-multimodal.md) | - | [Reranker](./voyageai/reranker.md) | | [Weaviate](./weaviate/index.md) | [Text](./weaviate/embeddings.md), [Multimodal](./weaviate/embeddings-multimodal.md) | - | - | | [xAI](./xai/index.md) | - | [Text](./xai/generative.md) | - | diff --git a/docs/weaviate/model-providers/morph/_category_.json b/docs/weaviate/model-providers/morph/_category_.json new file mode 100644 index 000000000..2acd1075a --- /dev/null +++ b/docs/weaviate/model-providers/morph/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Morph", + "position": 245.5 +} diff --git a/docs/weaviate/model-providers/morph/embeddings.md b/docs/weaviate/model-providers/morph/embeddings.md new file mode 100644 index 000000000..6909a8b74 --- /dev/null +++ b/docs/weaviate/model-providers/morph/embeddings.md @@ -0,0 +1,279 @@ +--- +title: Text Embeddings +sidebar_position: 20 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'morph', 'embeddings'] +--- + +# Morph Embeddings with Weaviate + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyConnect from '!!raw-loader!../_includes/provider.connect.py'; +import TSConnect from '!!raw-loader!../_includes/provider.connect.ts'; +import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py'; +import TSCode from '!!raw-loader!../_includes/provider.vectorizer.ts'; +import JavaV6Code from "!!raw-loader!/_includes/code/java-v6/src/test/java/ModelProvidersTest.java"; +import CSharpCode from "!!raw-loader!/_includes/code/csharp/ModelProvidersTest.cs"; + +Weaviate's integration with [Morph's API](https://docs.morphllm.com/) lets you access Morph-hosted embedding models directly from Weaviate. + +[Configure a Weaviate vector index](#configure-the-vectorizer) to use a Morph embedding model, and Weaviate generates embeddings for imports and searches automatically using your Morph API key. This is the *vectorizer*. + +At [import time](#data-import), Weaviate generates text object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings. + +![Embedding integration illustration](../_includes/integration_morph_embedding.png) + +:::caution Morph lists the Embedding API as legacy +Morph's own documentation labels the Embedding API as legacy and planned for deprecation. Check the current status in [Morph's documentation](https://docs.morphllm.com/) before you build on this integration. +::: + +## Requirements + +### Weaviate configuration + +Your Weaviate instance must have the `text2vec-morph` module enabled. The module is available in Weaviate `v1.32.6` and later. + +
+ For Weaviate Cloud (WCD) users + +This integration is enabled by default on Weaviate Cloud (WCD) instances. + +
+ +
+ For self-hosted users + +- Check the [cluster metadata](/deploy/configuration/status.md#cluster-metadata) to verify if the module is enabled. +- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate. + +
+ +### API credentials {#api-credentials} + +You must provide a Morph API key to Weaviate for this integration. Generate one in the [Morph dashboard](https://morphllm.com/) and supply it via one of: + +- Set the `MORPH_APIKEY` environment variable on the Weaviate server. +- Provide the `X-Openai-Api-Key` header at request time, as shown below. + +Weaviate builds Morph requests with its OpenAI-compatible client, so the request header is `X-Openai-Api-Key`. There is no Morph-specific header. A key provided in the header takes precedence over the server environment variable. + +:::caution The missing-key error names the wrong environment variable +When no key is available, Weaviate reports: + +``` +no api key found neither in request header: X-Openai-Api-Key nor in environment variable under OPENAI_APIKEY +``` + +The header name in that message is correct, but the environment variable name is not. This integration reads `MORPH_APIKEY`. Setting `OPENAI_APIKEY` does not make it work. +::: + + + + + + + + + + + + + + + + +:::note One header serves two integrations +`X-Openai-Api-Key` is also the header for the [OpenAI integration](../openai/embeddings.md). A single request therefore cannot carry different keys for the two integrations. If you use both in the same instance, set the server environment variables instead so each integration gets its own key. +::: + +## Configure the vectorizer + +[Configure a Weaviate index](../../manage-collections/vector-config.mdx#specify-a-vectorizer) to use a Morph embedding model by setting the vectorizer as follows: + + + + + + + + + + + + + + + + +import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; + +
+ Vectorization behavior + + + +
+ +### Vectorizer parameters + +- `model`: The Morph model id. Defaults to `morph-embedding-v3`. +- `baseURL`: The base URL prefix that requests are sent to. Any existing path is preserved when `endpoint` is appended. Defaults to `https://api.morphllm.com`. +- `endpoint`: The API path that Weaviate appends to the base URL. Defaults to `/v1/embeddings`. Set it if the service you target uses a different path. + +For how Weaviate combines `baseURL` and `endpoint` into a request URL, see [Header parameters](#header-parameters). + +:::info `endpoint` availability +Added in `v1.38.2` (backported to `v1.36.19` and `v1.37.10`). +::: + +Weaviate stores `baseURL` and `model` in the collection configuration even when you do not set them, because the module supplies a default for each. `endpoint` is different: it appears in the stored configuration only when you set it explicitly. If you read a collection back and see no `endpoint`, the default path applies. + +No `dimensions` parameter is sent, so the embedding dimension is always the model's native size. + +#### Example configuration + +The following examples set the Morph-specific options. Client libraries do not all expose the same options, so each example shows what that client supports. + + + + + + + + + + + + + + + + +## Header parameters + +You can provide the API key and the base URL at runtime through headers. Headers provided at request time take precedence over the collection configuration and over the server environment variable: + +- `X-Openai-Api-Key`: The Morph API key for this request. +- `X-Openai-Baseurl`: The base URL to use instead of the default. + +Provide the headers as shown in the [API credentials examples](#api-credentials) above. + +:::note How Weaviate builds the request URL + +Weaviate builds the request URL by appending the `endpoint` path (`/v1/embeddings` by default) to the base URL. The base URL supplies the scheme and host; `endpoint` supplies only the path. A value in `endpoint` cannot redirect requests to a different host. + +If a base URL already carries a path, that path is kept and the `endpoint` path is appended to it. + +There is no header that overrides `endpoint`. Set it in the collection configuration. + +::: + +:::note Error messages name the OpenAI API +Because Weaviate uses its OpenAI-compatible client for this integration, upstream failures are reported as `connection to: OpenAI API failed with status: ...` even when the request was sent to Morph. +::: + +## Data import + +After configuring the vectorizer, [import data](../../manage-objects/import.mdx) into Weaviate. Weaviate generates embeddings for text objects using [the configured model](#vectorizer-parameters). + +:::tip Re-use existing vectors +If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system. +::: + +## Searches + +Once the vectorizer is configured, Weaviate performs vector and hybrid searches using the specified Morph model. + +![Embedding integration at search illustration](../_includes/integration_morph_embedding_search.png) + +### Vector (near text) search {#vector-near-text-search} + +When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the configured Morph model and returns the most similar objects. + +### Hybrid search {#hybrid-search} + +When you perform a [hybrid search](../../search/hybrid.md), Weaviate fuses keyword and vector ranking. The text query is embedded with the configured Morph model; the keyword side uses Weaviate's inverted index. + +## References + +### Available models + +Weaviate does not restrict which model id you can set, so any model the Morph API accepts can be used. `morph-embedding-v3` is the default. Morph's [list models endpoint](https://docs.morphllm.com/api-reference/endpoint/models) returns the model ids your key can use. Check it before you rely on a model id, as availability and dimensions can change. + +## Further resources + +### Other integrations + +- [Weaviate model providers overview](../index.md) + +### Code examples + +Once the vectorizer is configured, Weaviate handles model inference transparently. The standard [client library how-tos](../../client-libraries/index.mdx) apply unchanged. No Morph-specific code is required at query or import time beyond the configuration shown above. + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/model-providers/morph/index.md b/docs/weaviate/model-providers/morph/index.md new file mode 100644 index 000000000..2220aa159 --- /dev/null +++ b/docs/weaviate/model-providers/morph/index.md @@ -0,0 +1,42 @@ +--- +title: Morph + Weaviate +sidebar_position: 10 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'morph'] +--- + + + +[Morph](https://morphllm.com/) serves code and text embedding models behind an OpenAI-compatible API. Weaviate integrates with Morph's embedding endpoint so you can vectorize and search data using Morph-hosted models directly from your Weaviate instance. + +:::caution Morph lists the Embedding API as legacy +Morph's own documentation labels the Embedding API as legacy and planned for deprecation. Check the current status in [Morph's documentation](https://docs.morphllm.com/) before you build on this integration. +::: + +## Integrations with Morph + +### Embedding models for vector search + +![Embedding integration illustration](../_includes/integration_morph_embedding.png) + +Morph exposes embedding models over an OpenAI-compatible `/v1/embeddings` API at `https://api.morphllm.com`. + +[Weaviate integrates with Morph's embedding models](./embeddings.md) through the `text2vec-morph` vectorizer module. Configure a vector index to use a Morph model and Weaviate generates embeddings for imports, vector searches, and hybrid searches automatically. + +[Morph embedding integration page](./embeddings.md) + +## Summary + +This integration lets you use Morph's hosted embedding models from Weaviate without managing inference infrastructure yourself. + +## Get started + +Generate an API key in the [Morph dashboard](https://morphllm.com/), then supply it to Weaviate through the `MORPH_APIKEY` environment variable or the `X-Openai-Api-Key` request header. The header name is shared with the OpenAI integration, because Morph requests are built by the same OpenAI-compatible client inside Weaviate. Then see the embedding integration page: + +- [Text Embeddings](./embeddings.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/model-providers/twelvelabs/_category_.json b/docs/weaviate/model-providers/twelvelabs/_category_.json new file mode 100644 index 000000000..6ca8d7718 --- /dev/null +++ b/docs/weaviate/model-providers/twelvelabs/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "TwelveLabs", + "position": 260 +} diff --git a/docs/weaviate/model-providers/twelvelabs/embeddings-multimodal.md b/docs/weaviate/model-providers/twelvelabs/embeddings-multimodal.md new file mode 100644 index 000000000..21f4cd182 --- /dev/null +++ b/docs/weaviate/model-providers/twelvelabs/embeddings-multimodal.md @@ -0,0 +1,396 @@ +--- +title: Multimodal Embeddings +description: "Weaviate's integration with TwelveLabs' APIs allows you to access their models' capabilities directly from Weaviate." +sidebar_position: 25 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'twelvelabs', 'embeddings'] +--- + +# TwelveLabs Multimodal Embeddings with Weaviate + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyConnect from '!!raw-loader!../_includes/provider.connect.py'; +import PyCode from '!!raw-loader!../_includes/provider.vectorizer.py'; + +Weaviate's integration with TwelveLabs' APIs allows you to access their models' capabilities directly from Weaviate. + +[Configure a Weaviate vector index](#configure-the-vectorizer) to use a TwelveLabs embedding model, and Weaviate will generate embeddings for various operations using the specified model and your TwelveLabs API key. This feature is called the *vectorizer*. + +At [import time](#data-import), Weaviate generates multimodal object embeddings and saves them into the index. For [vector](#vector-near-text-search) and [hybrid](#hybrid-search) search operations, Weaviate converts text queries into embeddings. [Multimodal search operations](#vector-near-image-search) are also supported. + +:::caution Text and images only +TwelveLabs is best known for video understanding, but this integration vectorizes **text and images only**. Video and audio are not supported. + +The vectorizer reads only the `textFields` and `imageFields` settings, and only `nearText` and `nearImage` search operations are available. Weaviate stores a `videoFields` entry if you add one to a collection definition, but nothing reads it, so it has no effect. +::: + +![Embedding integration illustration](../_includes/integration_twelvelabs_embedding.png) + +## Requirements + +### Weaviate configuration + +Your Weaviate instance must be configured with the TwelveLabs vectorizer integration (`multi2vec-twelvelabs`) module. + +:::info Added in `v1.38.9` +This integration is available in Weaviate `v1.38.9`, `v1.39.0` and later. +::: + +
+ For Weaviate Cloud (WCD) users + +This integration is enabled by default on Weaviate Cloud (WCD) instances. + +
+ +
+ For self-hosted users + +- Check the [cluster metadata](/deploy/configuration/status.md#cluster-metadata) to verify if the module is enabled. +- Follow the [how-to configure modules](../../configuration/modules.md) guide to enable the module in Weaviate. + +
+ +### API credentials + +You must provide a valid TwelveLabs API key to Weaviate for this integration. Go to [TwelveLabs](https://www.twelvelabs.io/) to sign up and obtain an API key. + +Provide the API key to Weaviate using one of the following methods: + +- Set the `TWELVELABS_APIKEY` environment variable that is available to Weaviate. +- Provide the API key at runtime, as shown in the examples below. + + + + + + + + + +```bash +curl http://localhost:8080/v1/graphql \ + -H "Content-Type: application/json" \ + -H "X-Twelvelabs-Api-Key: $TWELVELABS_APIKEY" \ + -H "X-Twelvelabs-Baseurl: https://api.twelvelabs.io/v1.3" \ + -d '{"query": "{ Get { DemoCollection(nearText: {concepts: [\"A holiday film\"]}, limit: 2) { title } } }"}' +``` + + + + + +The `X-Twelvelabs-Baseurl` header is optional. It overrides the base URL that is set in the collection definition for the duration of the request. + +## Configure the vectorizer + +[Configure a Weaviate index](../../manage-collections/vector-config.mdx#specify-a-vectorizer) as follows to use a TwelveLabs embedding model. + +Name the properties that hold your text in `textFields`, and the properties that hold your base64 encoded images in `imageFields`. Set at least one of the two. A collection that names no fields cannot produce a vector, and inserts into it fail with a `more than one embedding found for object` error. + + + + + + + + +```bash +curl -X POST http://localhost:8080/v1/schema \ + -H "Content-Type: application/json" \ + -d '{ + "class": "DemoCollection", + "properties": [ + {"name": "title", "dataType": ["text"]}, + {"name": "poster", "dataType": ["blob"]} + ], + "vectorConfig": { + "title_vector": { + "vectorizer": { + "multi2vec-twelvelabs": { + "textFields": ["title"], + "imageFields": ["poster"], + "weights": { + "textFields": [0.1], + "imageFields": [0.9] + } + } + }, + "vectorIndexType": "hnsw" + } + } + }' +``` + + + + + +:::info Client availability +A typed configuration API for this integration is currently available in the Python client only. If `Configure.Vectors.multi2vec_twelvelabs` is missing from your installation, upgrade to the latest Python client version. + +With any other client library, configure the vectorizer by sending the collection definition as shown in the cURL example above, or by passing the equivalent module configuration map that your client accepts. Data import and search operations are not specific to this integration and work with every client library. +::: + +### Select a model + +You can specify one of the [available models](#available-models) for the vectorizer to use, as shown in the following configuration example. + + + + + + + + +```bash +curl -X POST http://localhost:8080/v1/schema \ + -H "Content-Type: application/json" \ + -d '{ + "class": "DemoCollection", + "properties": [ + {"name": "title", "dataType": ["text"]}, + {"name": "poster", "dataType": ["blob"]} + ], + "vectorConfig": { + "title_vector": { + "vectorizer": { + "multi2vec-twelvelabs": { + "textFields": ["title"], + "imageFields": ["poster"], + "model": "marengo3.0" + } + }, + "vectorIndexType": "hnsw" + } + } + }' +``` + + + + + +You can [specify](#vectorizer-parameters) one of the [available models](#available-models) for Weaviate to use. The [default model](#available-models) is used if no model is specified. + +import VectorizationBehavior from '/_includes/vectorization.behavior.mdx'; + +
+ Vectorization behavior + + + +
+ +### Vectorizer parameters + +The following examples show how to configure TwelveLabs-specific options. + + + + + + + + +```bash +curl -X POST http://localhost:8080/v1/schema \ + -H "Content-Type: application/json" \ + -d '{ + "class": "DemoCollection", + "properties": [ + {"name": "title", "dataType": ["text"]}, + {"name": "poster", "dataType": ["blob"]} + ], + "vectorConfig": { + "title_vector": { + "vectorizer": { + "multi2vec-twelvelabs": { + "textFields": ["title"], + "imageFields": ["poster"], + "weights": { + "textFields": [0.1], + "imageFields": [0.9] + }, + "model": "marengo3.0", + "baseURL": "https://api.twelvelabs.io/v1.3" + } + }, + "vectorIndexType": "hnsw" + } + } + }' +``` + + + + + +The collection definition accepts the following settings: + +| Setting | Description | +| --- | --- | +| `textFields` | Names of the `text` and `text[]` properties to vectorize. Each element of a `text[]` property is vectorized separately. | +| `imageFields` | Names of the properties that hold base64 encoded images, typically `blob` properties. A `text[]` property listed here is ignored. | +| `weights` | Relative weights for combining the field vectors, given as `textFields` and `imageFields` arrays. Each array must have the same number of entries as the field list it weights. The weights are normalized so that they sum to 1. If no weights are set, all fields are weighted equally. | +| `model` | The model to use. The default is `marengo3.0`. | +| `baseURL` | The base URL of the TwelveLabs API. The default is `https://api.twelvelabs.io/v1.3`. | + +In the Python client, these settings are named `text_fields`, `image_fields`, `model` and `base_url`. Weights are set per field with `Multi2VecField(name=..., weight=...)`. + +:::note Settings that have no effect +Weaviate writes a `vectorizeClassName` setting into every collection that uses this integration, but this integration does not read it. Its value does not change the vectors that are produced, and the collection name is never included in the vectorized text. + +The per-property `skip` and `vectorizePropertyName` settings also have no effect here. Property selection is determined only by `textFields` and `imageFields` membership, and property names are never vectorized. +::: + +For further details on model parameters, see the [TwelveLabs documentation](https://docs.twelvelabs.io/). + +## Data import + +After configuring the vectorizer, [import data](../../manage-objects/import.mdx) into Weaviate. Weaviate generates embeddings for text and image objects using the specified model. + +Provide image data as a base64 encoded string. A `data:;base64,` prefix is accepted and stripped before decoding. A property that is listed in `imageFields` but does not hold valid base64 data fails the import with a `decode base64 image` error. + + + + + + + + + +:::warning This integration is not rate limited +Weaviate does not throttle its requests to TwelveLabs for this integration, and it does not read rate limit response headers. It sends one request per text value and one request per image, and it processes batches of ten objects in parallel, so a large import can produce a high request rate. + +If TwelveLabs rejects a request, the error surfaces as a failed import for that object and is not retried. Pace large imports from the client side, for example by importing in smaller batches. +::: + +:::tip Re-use existing vectors +If you already have a compatible model vector available, you can provide it directly to Weaviate. This can be useful if you have already generated embeddings using the same model and want to use them in Weaviate, such as when migrating data from another system. +::: + +## Searches + +Once the vectorizer is configured, Weaviate will perform vector and hybrid search operations using the specified TwelveLabs model. + +![Embedding integration at search illustration](../_includes/integration_twelvelabs_embedding_search.png) + +The examples below use the Python client. Search operations are not specific to this integration, so see the [How-to: Query & Search](../../search/index.mdx) guides for the equivalent examples in the other client libraries. + +### Vector (near text) search + +When you perform a [vector search](../../search/similarity.md#search-with-text), Weaviate converts the text query into an embedding using the specified model and returns the most similar objects from the database. + +The query below returns the `n` most similar objects from the database, set by `limit`. + + + + + + + + + +### Hybrid search + +:::info What is a hybrid search? +A hybrid search performs a vector search and a keyword (BM25) search, before [combining the results](../../search/hybrid.md#change-the-fusion-method) to return the best matching objects from the database. +::: + +When you perform a [hybrid search](../../search/hybrid.md), Weaviate converts the text query into an embedding using the specified model and returns the best scoring objects from the database. + +The query below returns the `n` best scoring objects from the database, set by `limit`. + + + + + + + + + +### Vector (near image) search + +When you perform a [near image search](../../search/similarity.md#search-with-image), Weaviate converts the query into an embedding using the specified model and returns the most similar objects from the database. + +To perform a near image search, convert the image query into a base64 string and pass it to the search query. + +The query below returns the `n` most similar objects to the input image from the database, set by `limit`. + + + + + + + + + +## References + +### Available models + +The default model is `marengo3.0`, which produces 512-dimensional vectors. + +Weaviate does not validate the model name, so you can set any model that the TwelveLabs embedding endpoint accepts for your account. Weaviate does not publish the list of accepted names; see the [TwelveLabs documentation on creating embeddings](https://docs.twelvelabs.io/v1.3/docs/guides/create-embeddings) for the models that are currently available. + +## Further resources + +### Code examples + +Once the integrations are configured at the collection, the data management and search operations in Weaviate work identically to any other collection. See the following model-agnostic examples: + +- The [How-to: Manage collections](../../manage-collections/index.mdx) and [How-to: Manage objects](../../manage-objects/index.mdx) guides show how to perform data operations (i.e. create, read, update, delete collections and objects within them). +- The [How-to: Query & Search](../../search/index.mdx) guides show how to perform search operations (i.e. vector, keyword, hybrid) as well as retrieval augmented generation. + +### External resources + +- [TwelveLabs documentation](https://docs.twelvelabs.io/) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/model-providers/twelvelabs/index.md b/docs/weaviate/model-providers/twelvelabs/index.md new file mode 100644 index 000000000..2f1e7c1d0 --- /dev/null +++ b/docs/weaviate/model-providers/twelvelabs/index.md @@ -0,0 +1,46 @@ +--- +title: TwelveLabs + Weaviate +sidebar_position: 10 +image: og/docs/model-provider-integrations.jpg +# tags: ['model providers', 'twelvelabs'] +--- + + + +TwelveLabs builds multimodal understanding models. Weaviate integrates with the TwelveLabs embedding API, so you can vectorize your data and your queries with a TwelveLabs model without leaving the Weaviate Database. + +:::caution Text and images only +TwelveLabs is best known for video understanding, but this Weaviate integration works with **text and images only**. Video and audio are not supported. +::: + +## Integrations with TwelveLabs + +### Embedding models for vector search + +![Embedding integration illustration](../_includes/integration_twelvelabs_embedding.png) + +TwelveLabs' embedding models place text and images in a shared vector space, so that a text query can retrieve images and vice versa. + +[Weaviate integrates with TwelveLabs' embedding models](./embeddings-multimodal.md) to enable seamless vectorization of data. This integration allows users to perform semantic and hybrid search operations without the need for additional preprocessing or data transformation steps. + +[TwelveLabs multimodal embedding integration page](./embeddings-multimodal.md) + +## Summary + +This integration enables developers to use TwelveLabs' multimodal embedding models within Weaviate. + +In turn, it simplifies the process of building AI-driven applications to speed up your development process, so that you can focus on creating innovative solutions. + +## Get started + +You must provide a valid TwelveLabs API key to Weaviate for this integration. Go to [TwelveLabs](https://www.twelvelabs.io/) to sign up and obtain an API key. + +Then, go to the relevant integration page to learn how to configure Weaviate with the TwelveLabs models and start using them in your applications. + +- [Multimodal Embeddings](./embeddings-multimodal.md) + +## Questions and feedback + +import DocsFeedback from '/_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/release-notes/index.md b/docs/weaviate/release-notes/index.md index ca97aaac2..1bbd63dd4 100644 --- a/docs/weaviate/release-notes/index.md +++ b/docs/weaviate/release-notes/index.md @@ -17,6 +17,11 @@ import QuickLinks from "/src/components/QuickLinks"; export const pythonCardsData = [ { +title: "v1.39", +link: "https://github.com/weaviate/weaviate/releases/tag/v1.39.0", +icon: "fa fa-tags", +}, +{ title: "v1.38", link: "https://github.com/weaviate/weaviate/releases/tag/v1.38.0", icon: "fa fa-tags", @@ -36,11 +41,6 @@ title: "v1.35", link: "https://weaviate.io/blog/weaviate-1-35-release", icon: "fa fa-tags", }, -{ -title: "v1.34", -link: "https://weaviate.io/blog/weaviate-1-34-release", -icon: "fa fa-tags", -}, ]; diff --git a/docs/weaviate/search/bm25.md b/docs/weaviate/search/bm25.md index 061990ce0..0f24fd76d 100644 --- a/docs/weaviate/search/bm25.md +++ b/docs/weaviate/search/bm25.md @@ -16,7 +16,7 @@ import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-b import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchKeywordTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchKeywordTest.cs"; import GQLCode from '!!raw-loader!/\_includes/code/howto/search.bm25.gql.py'; -import BoostPreview from '/_includes/feature-notes/boost.mdx'; +import BoostNote from '/_includes/feature-notes/boost.mdx'; `Keyword` search, also called "BM25 (Best match 25)" or "sparse vector" search, returns objects that have the highest BM25F scores. @@ -99,7 +99,7 @@ import SearchOperators from '/_includes/feature-notes/search-operators.mdx'; -Search operators define the minimum number of query [tokens](#set-tokenization) that must be present within a single searched property for an object to be returned. The options are `and`, or `or` (default). +Search operators define how many of the query [tokens](#set-tokenization) must match, and whether they must all match within a single searched property. The options are `or` (default), `and`, and `and_cross`. ### `or` @@ -179,6 +179,48 @@ With the `and` operator, the search returns objects where all tokens in the sear
+### `and_cross` + +:::info Added in `v1.38.8` +::: + +With the `and_cross` operator, every token in the search string must be matched by at least one of the searched properties, but the tokens do not all have to occur in the same property. An object whose title matches one token and whose body matches the rest is a match for `and_cross`, and is not a match for `and`. + +Because it relaxes the single-property requirement, `and_cross` returns every object that `and` returns, and usually more. + +:::caution All searched properties must be configured alike + +`and_cross` requires every searched property to share the same tokenization and the same analyzer settings, which means the tokenizer, accent folding and its exceptions, and the stopword preset. If they differ, the query fails with an error instead of returning fewer results: + +``` +OPERATOR_AND_CROSS requires all searched properties to share the same tokenization and analyzer settings +``` + +If a collection mixes tokenizations, restrict the search to a compatible set of properties. See [Search on selected properties only](#search-on-selected-properties-only). + +::: + +The examples below use Python and GraphQL, because `and_cross` is currently available in the Python client and not yet in the TypeScript, Go, Java, or C# clients. + + + + + + + + + + ## Retrieve BM25F scores You can retrieve the BM25F `score` values for each returned object. @@ -767,7 +809,7 @@ Set the tokenization method to `trigram` at the property level when creating you ## Soft-rank with Boost - + Keyword (BM25) queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. diff --git a/docs/weaviate/search/boost.md b/docs/weaviate/search/boost.md index b0db2da40..cb6c3404e 100644 --- a/docs/weaviate/search/boost.md +++ b/docs/weaviate/search/boost.md @@ -10,9 +10,9 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; import PyCode from '!!raw-loader!/_includes/code/howto/search.boost.py'; -import BoostPreview from '/_includes/feature-notes/boost.mdx'; +import BoostNote from '/_includes/feature-notes/boost.mdx'; - + **Boost** soft-ranks search results: it promotes or demotes matching documents without removing them from the result set. Matching documents move up. Non-matching documents stay in the results but rank lower. diff --git a/docs/weaviate/search/hybrid.md b/docs/weaviate/search/hybrid.md index bbf729067..50fecf2f4 100644 --- a/docs/weaviate/search/hybrid.md +++ b/docs/weaviate/search/hybrid.md @@ -16,7 +16,8 @@ import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-h import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchHybridTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchHybridTest.cs"; import GQLCode from '!!raw-loader!/\_includes/code/howto/search.hybrid.gql.py'; -import BoostPreview from '/_includes/feature-notes/boost.mdx'; +import MMRPyCode from '!!raw-loader!/\_includes/code/howto/search.similarity.mmr.py'; +import BoostNote from '/_includes/feature-notes/boost.mdx'; `Hybrid` search combines the results of a vector search and a keyword (BM25F) search by fusing the two result sets. @@ -386,7 +387,9 @@ import SearchOperators from '/_includes/feature-notes/search-operators.mdx'; -Keyword (BM25) search operators define the minimum number of query [tokens](#tokenization) that must be present within a single searched property for an object to be returned. The options are `and`, or `or` (default). +Keyword (BM25) search operators define how many of the query [tokens](#tokenization) must match, and whether they must all match within a single searched property. The options are `or` (default), `and`, and `and_cross` (available from `v1.38.8`). + +The keyword leg of a hybrid query accepts the same operators as a standalone keyword search. For `and_cross`, which matches every token across the searched properties combined, see [BM25 search: `and_cross`](./bm25.md#and_cross). ### `or` @@ -1043,7 +1046,7 @@ import TokenizationNote from '/\_includes/tokenization.mdx' ## Soft-rank with Boost - + Hybrid queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. @@ -1051,6 +1054,38 @@ The boost runs once over the **fused** hybrid result. The BM25 and vector sub-se See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. +## Diversity selection (MMR) + +:::info Added in `v1.38.6` +::: + +Hybrid search fuses a keyword result set and a vector result set, which often means the top of the fused list is a cluster of near-duplicates. **Maximum Marginal Relevance (MMR)** reranks that list to balance relevance with diversity, so that each selected object adds something new to the result set. + +Diversity selection runs after fusion. Both search legs run first, their results are fused with the configured `alpha` and fusion method, and the diversity pass then picks a diverse subset of the fused candidates. + +The examples in this section are Python only, because diversity selection is currently available in the Python client and not yet in the TypeScript, Go, Java, or C# clients. There is no GraphQL equivalent. + + + + + + + +Important notes: + +- **Top-level only**: set diversity selection on the hybrid query itself. Setting it on a sub-search is rejected with an error. +- **Two limits**: the query's top-level `limit` is the candidate window that gets diversified, and the diversity `limit` is the number of results returned. The diversity `limit` must be at least `1` and no larger than the query `limit`. +- **Ordering**: results come back in MMR order, not fused-score order. +- **Pagination**: `offset` moves the candidate window, so it must advance by the query `limit`, not by the number of returned objects. Weaviate does not validate this, and getting it wrong silently repeats some objects across pages while skipping others. See [Pagination](./similarity.md#pagination). +- **Not supported**: multi-vector collections. Weaviate rejects these queries with an error. + +For the parameters, the relevance and diversity trade-off, and vector search examples, see [Diversity selection (MMR)](./similarity.md#diversity-selection-mmr). + ## Related pages - [Connect to Weaviate](/weaviate/connections/index.mdx) diff --git a/docs/weaviate/search/rerank.md b/docs/weaviate/search/rerank.md index 7ff1139cc..c9f9ea311 100644 --- a/docs/weaviate/search/rerank.md +++ b/docs/weaviate/search/rerank.md @@ -16,7 +16,7 @@ import SimilarityPyCode from '!!raw-loader!/_includes/code/howto/search.similari import SimilarityPyCodeV3 from '!!raw-loader!/_includes/code/howto/search.similarity-v3.py'; import SimilarityTSCode from '!!raw-loader!/_includes/code/howto/search.similarity.ts'; import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/mainpkg/search-rerank_test.go'; -import BoostPreview from '/_includes/feature-notes/boost.mdx'; +import BoostNote from '/_includes/feature-notes/boost.mdx'; Reranking modules reorder the search result set according to a different set of criteria or a different (e.g. more expensive) algorithm. @@ -199,7 +199,7 @@ The response should look like this: ## Soft-rank with Boost - + For lightweight result reordering based on filters, property values, or time / numeric decay (without calling an external rerank model), use [Boost](./boost.md). Rerank and Boost can be used independently. Pick rerank when you need a smarter model to re-rank the top-N, and Boost when you want to bias by simple signals already on the objects. diff --git a/docs/weaviate/search/similarity.md b/docs/weaviate/search/similarity.md index 86bc718ad..ebabf4fcd 100644 --- a/docs/weaviate/search/similarity.md +++ b/docs/weaviate/search/similarity.md @@ -15,7 +15,7 @@ import TSCode from '!!raw-loader!/\_includes/code/howto/search.similarity.ts'; import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-similarity_test.go'; import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchSimilarityTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchSimilarityTest.cs"; -import BoostPreview from '/_includes/feature-notes/boost.mdx'; +import BoostNote from '/_includes/feature-notes/boost.mdx'; Vector search returns the objects with most similar vectors to that of the query. @@ -696,7 +696,7 @@ Add the `diversity_selection` parameter to any vector search query: | Parameter | Type | Description | | :-------- | :---- | :------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `limit` | int | Number of results to return after MMR reranking. Must be less than or equal to the query's top-level `limit` (the candidate set size). | +| `limit` | int | Number of results to return after MMR reranking. Must be at least `1` and no larger than the query's top-level `limit` (the candidate set size). Weaviate returns an error if it is omitted or out of range. | | `balance` | float | Controls the relevance-diversity trade-off (0.0–1.0). `0.0` = pure diversity, `0.5` = balanced, `1.0` = pure relevance (equivalent to standard search). | + Candidate window, worked example, and deep pages + +A diversified query works with two limits: + +- the query's top-level `limit` is the **candidate window** that gets diversified, and +- the diversity `limit` is the **page size**, or how many objects come back. + +Each page is taken from the slice `[offset, offset + limit)` of the relevance-ranked results, so `offset` must advance by the query `limit`: + + + +The usual pagination idiom, adding the number of returned objects to `offset`, is the failing case here. The page size is always smaller than or equal to the candidate window, so consecutive windows overlap. With a query `limit` of `10` and a diversity `limit` of `3`, advancing `offset` by `3` reads the windows `[0:10]`, `[3:13]`, and `[6:16]`, which is how objects come back twice while others are skipped. + +When diversity selection is combined with a boost or with hybrid search, deep pages are also not stable slices of one fixed ranking. Both the boost pool and the two hybrid search legs fetch more candidates as `offset` grows, so which objects can reach a given page depends on how deep you have paged. + +
+ ## Soft-rank with Boost - + Vector search queries accept an optional `boost` argument that promotes or demotes matching documents without removing them. This is useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. diff --git a/docusaurus.config.js b/docusaurus.config.js index 0d053915a..6450c6428 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -64,7 +64,7 @@ const config = { cdn: "https://cdn.jsdelivr.net/npm/@scalar/api-reference@1.49.0", configuration: { spec: { - url: "https://raw.githubusercontent.com/weaviate/weaviate/v1-38/openapi-for-docs/openapi-specs/schema.json", + url: "https://raw.githubusercontent.com/weaviate/weaviate/v1-39/openapi-for-docs/openapi-specs/schema.json", }, hideModels: true, showSidebar: true, diff --git a/netlify.toml b/netlify.toml index eea05e1de..82a8b2e25 100644 --- a/netlify.toml +++ b/netlify.toml @@ -506,6 +506,11 @@ from = "/weaviate/configuration/persistence" to = "/deploy/configuration/persistence" status = 301 +[[redirects]] +from = "/weaviate/configuration/hnsw-snapshots" +to = "/weaviate/concepts/storage#hnsw-snapshots" +status = 301 + [[redirects]] from = "/weaviate/configuration/monitoring" to = "/deploy/configuration/monitoring" diff --git a/pyproject.toml b/pyproject.toml index 657b44ece..17b920ae8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "requests>=2.32.3", "tqdm>=4.67.1", "weaviate-agents>=1.7.0", - "weaviate-client==4.22.0", + "weaviate-client==4.23.0", "weaviate-demo-datasets>=0.8.1", "weaviate-engram>=0.3.0", "anthropic>=0.40.0", diff --git a/sidebars.js b/sidebars.js index 05a5ff010..d6c16b846 100644 --- a/sidebars.js +++ b/sidebars.js @@ -238,6 +238,16 @@ const sidebars = { "weaviate/model-providers/databricks/generative", ], }, + { + type: "category", + label: "DeepSeek", + className: "sidebar-item", + link: { + type: "doc", + id: "weaviate/model-providers/deepseek/index", + }, + items: ["weaviate/model-providers/deepseek/generative"], + }, { type: "category", label: "DigitalOcean", @@ -310,6 +320,16 @@ const sidebars = { "weaviate/model-providers/mistral/generative", ], }, + { + type: "category", + label: "Morph", + className: "sidebar-item", + link: { + type: "doc", + id: "weaviate/model-providers/morph/index", + }, + items: ["weaviate/model-providers/morph/embeddings"], + }, { type: "category", label: "NVIDIA", @@ -364,6 +384,18 @@ const sidebars = { "weaviate/model-providers/openai-azure/generative", ], }, + { + type: "category", + label: "TwelveLabs", + className: "sidebar-item", + link: { + type: "doc", + id: "weaviate/model-providers/twelvelabs/index", + }, + items: [ + "weaviate/model-providers/twelvelabs/embeddings-multimodal", + ], + }, { type: "category", label: "VoyageAI", @@ -533,7 +565,6 @@ const sidebars = { "weaviate/configuration/compression/multi-vectors", ], }, - "weaviate/configuration/hnsw-snapshots", "weaviate/configuration/modules", { type: "doc", diff --git a/tests/docker-compose-anon-2.yml b/tests/docker-compose-anon-2.yml index 658c43a83..8567892c7 100644 --- a/tests/docker-compose-anon-2.yml +++ b/tests/docker-compose-anon-2.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8090:8080 - 50061:50051 diff --git a/tests/docker-compose-anon-bind.yml b/tests/docker-compose-anon-bind.yml index 094e83b74..0bc137c0b 100644 --- a/tests/docker-compose-anon-bind.yml +++ b/tests/docker-compose-anon-bind.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8380:8080 - 50351:50051 diff --git a/tests/docker-compose-anon-clip.yml b/tests/docker-compose-anon-clip.yml index 5349933c2..3968a8867 100644 --- a/tests/docker-compose-anon-clip.yml +++ b/tests/docker-compose-anon-clip.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8280:8080 - 50251:50051 diff --git a/tests/docker-compose-anon-offload.yml b/tests/docker-compose-anon-offload.yml index a4ef8c56b..fbc06efc9 100644 --- a/tests/docker-compose-anon-offload.yml +++ b/tests/docker-compose-anon-offload.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8080:8080 - 50051:50051 diff --git a/tests/docker-compose-anon.yml b/tests/docker-compose-anon.yml index 1ff802b26..06ca2b391 100644 --- a/tests/docker-compose-anon.yml +++ b/tests/docker-compose-anon.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8080:8080 - 50051:50051 diff --git a/tests/docker-compose-rbac.yml b/tests/docker-compose-rbac.yml index 5ea8add68..8e542e8b8 100644 --- a/tests/docker-compose-rbac.yml +++ b/tests/docker-compose-rbac.yml @@ -7,7 +7,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8580:8080 - 50551:50051 diff --git a/tests/docker-compose-three-nodes.yml b/tests/docker-compose-three-nodes.yml index a9bf30068..1634de0d3 100644 --- a/tests/docker-compose-three-nodes.yml +++ b/tests/docker-compose-three-nodes.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8180:8080" @@ -36,7 +36,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8181:8080" @@ -65,7 +65,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 restart: on-failure:0 ports: - "8182:8080" diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 5f56fcd80..487b02a5c 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.38.0 + image: cr.weaviate.io/semitechnologies/weaviate:1.39.0 ports: - 8099:8080 - 50052:50051 diff --git a/uv.lock b/uv.lock index b0f0e87f5..bb6093c5a 100644 --- a/uv.lock +++ b/uv.lock @@ -1826,21 +1826,21 @@ wheels = [ [[package]] name = "weaviate-agents" -version = "1.7.0" +version = "1.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx-sse" }, { name = "rich" }, { name = "weaviate-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b8/99/750d7d2e33f610bad0d4b1d77e2a7f0c73dbab5459876fa51c7a4a440a3a/weaviate_agents-1.7.0.tar.gz", hash = "sha256:f033085cbde7123424f5579dfab553df5e0128d395dfea3b58c5aea6602340cb", size = 111884, upload-time = "2026-07-27T08:31:58.622Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/fa/c392cbad0cd088070e9200e12a13d6cb3b8f33ee68e2afddec405c58e24d/weaviate_agents-1.8.0.tar.gz", hash = "sha256:878a4e0892ba028417389abf1caaea5947e8b76c6edcd706946c06093239bcfa", size = 112358, upload-time = "2026-08-11T13:46:50.144Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/d3/30952e1a4ca1556beb73e756d1f514c0bd62db53c21bdc65cdd6c5c6b78e/weaviate_agents-1.7.0-py3-none-any.whl", hash = "sha256:052e37ad9114424149b29deb8ba96159fce69f54e45874a4d6e42b1831643931", size = 54505, upload-time = "2026-07-27T08:31:57.755Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/97060d755c6e88b65371a13791704497841543b2267c27feef597e468d34/weaviate_agents-1.8.0-py3-none-any.whl", hash = "sha256:40b93991bba26f3931c105e3d6f2c71604f3d74f7801a6e9382562751cc9b3b3", size = 54972, upload-time = "2026-08-11T13:46:49.059Z" }, ] [[package]] name = "weaviate-client" -version = "4.22.0" +version = "4.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "authlib" }, @@ -1851,9 +1851,9 @@ dependencies = [ { name = "pydantic" }, { name = "validators" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/2a/73cf7d6c7c6aa638738dfcb0d318e0404aab3c0673f82a1a4d89455b21a5/weaviate_client-4.22.0.tar.gz", hash = "sha256:0c50fbef546a522262a87d1138cde0509c7a8a48e702e967be33472e9f7fbae3", size = 860126, upload-time = "2026-06-18T06:08:30.202Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/9f/8bfe42e3c0693afa321fb712c0a1f1ff2b886059db33543120f6953dde8e/weaviate_client-4.23.0.tar.gz", hash = "sha256:19cc336b4c9e58f06cf01cb9aff8f6d8c0fe747f2e80192a9871c63823e8b18d", size = 873028, upload-time = "2026-08-13T14:23:08.112Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/a3/27353ea3fbf9e7d4f03375176a838c632d009f5167d801adcbb5f6d3bd07/weaviate_client-4.22.0-py3-none-any.whl", hash = "sha256:ff2dbc8d1fc25739942402c22d1aba2350a16ba4a6b6ed0ba140689c70adf1d9", size = 652691, upload-time = "2026-06-18T06:08:28.622Z" }, + { url = "https://files.pythonhosted.org/packages/15/aa/8cafeb1c09901e8d220b00b771ffea35e5ea51db7d384dc85eff6cf667dc/weaviate_client-4.23.0-py3-none-any.whl", hash = "sha256:f17c645d22e1787e8fa4640e20fcfccdee39c81327af410c544a24c21d419e59", size = 656174, upload-time = "2026-08-13T14:23:06.583Z" }, ] [[package]] @@ -1908,7 +1908,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.3" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "weaviate-agents", specifier = ">=1.7.0" }, - { name = "weaviate-client", specifier = "==4.22.0" }, + { name = "weaviate-client", specifier = "==4.23.0" }, { name = "weaviate-demo-datasets", specifier = ">=0.8.1" }, { name = "weaviate-engram", specifier = ">=0.3.0" }, ]